From 821c4211a528f9dbf2943913f5f2607b544e12bc Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 8 Jun 2022 12:11:40 +0530 Subject: [PATCH 01/15] Replace uses of go-playground/assert/v2 with testify/assert for parity. Signed-off-by: Vighnesh Shenoy --- go.mod | 1 - pkg/scalers/azure/azure_eventhub_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d43b7389f7b..c4ca0af0302 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/dysnix/predictkube-proto v0.0.0-20211223141524-d309509b6b5f github.com/elastic/go-elasticsearch/v7 v7.17.1 github.com/go-logr/logr v1.2.3 - github.com/go-playground/assert/v2 v2.0.1 github.com/go-playground/validator/v10 v10.11.0 github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql v1.6.0 diff --git a/pkg/scalers/azure/azure_eventhub_test.go b/pkg/scalers/azure/azure_eventhub_test.go index 76aeb4be1f4..fbaa726dedc 100644 --- a/pkg/scalers/azure/azure_eventhub_test.go +++ b/pkg/scalers/azure/azure_eventhub_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/go-playground/assert/v2" + "github.com/stretchr/testify/assert" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) From f5204c3d648803a8df78d7500115bd15818d3178 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 8 Jun 2022 13:46:45 +0530 Subject: [PATCH 02/15] Update test runner scripts. Signed-off-by: Vighnesh Shenoy --- tests/run-all.sh | 33 +++++++++++++++++++++++++-------- tests/run-arm-smoke-tests.sh | 25 ++++++++++++++++++++----- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/tests/run-all.sh b/tests/run-all.sh index 47dcc60f970..d05ca2c4d18 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -1,7 +1,9 @@ #! /bin/bash set -eu -E2E_REGEX=${E2E_TEST_REGEX:-*.test.ts} +# TODO - Remove TypeScript regex after all tests have been migrated to Go. +E2E_REGEX_GO="${E2E_TEST_REGEX}_*test.go" +E2E_REGEX_TS="${E2E_TEST_REGEX}-*test.ts" DIR=$(dirname "$0") cd $DIR @@ -15,21 +17,29 @@ counter=0 executed_count=0 function run_setup { - ./node_modules/.bin/ava setup.test.ts + go test -v setup_test.go } function run_tests { counter=0 # randomize tests order using shuf - for test_case in $(find scalers -name "$E2E_REGEX" | shuf) + # TODO - Remove TypeScript regex after all tests have been migrated to Go. + for test_case in $(find . -name "$E2E_REGEX_GO" -o -name "$E2E_REGEX_TS" | shuf) do - if [[ $test_case != *.test.ts ]] # Skip helper files + if [[ $test_case != *_test.go && $test_case != *.test.ts ]] # Skip helper files then continue fi counter=$((counter+1)) - ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & + # TODO - Remove condition after all tests have been migrated to Go. + if [[ $test_case == *_test.go ]] + then + go test -v $test_case > "${test_case}.log" 2>&1 & + else + ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & + fi + pid=$! echo "Running $test_case with pid: $pid" pids+=($pid) @@ -62,7 +72,14 @@ function run_tests { for test_case in "${retry_lookup[@]}" do counter=$((counter+1)) - ./node_modules/.bin/ava $test_case > "${test_case}.retry.log" 2>&1 & + # TODO - Remove condition after all tests have been migrated to Go. + if [[ $test_case == *_test.go ]] + then + go test -v $test_case > "${test_case}.retry.log" 2>&1 & + else + ./node_modules/.bin/ava $test_case > "${test_case}.retry.log" 2>&1 & + fi + pid=$! echo "Rerunning $test_case with pid: $pid" pids+=($pid) @@ -94,7 +111,7 @@ function wait_for_jobs { } function print_logs { - for test_log in $(find scalers -name "*.log") + for test_log in $(find . -name "*.log") do echo ">>> $test_log <<<" cat $test_log @@ -114,7 +131,7 @@ function print_logs { } function run_cleanup { - ./node_modules/.bin/ava cleanup.test.ts + go test -v cleanup_test.go } function print_failed { diff --git a/tests/run-arm-smoke-tests.sh b/tests/run-arm-smoke-tests.sh index b747869f18b..d596e3fb56f 100755 --- a/tests/run-arm-smoke-tests.sh +++ b/tests/run-arm-smoke-tests.sh @@ -4,6 +4,7 @@ set -eu DIR=$(dirname "$0") cd $DIR +# TODO - Replace with Go tests. test_files=( "kubernetes-workload.test.ts" "activemq.test.ts" @@ -18,7 +19,7 @@ failed_lookup=() counter=0 function run_setup { - ./node_modules/.bin/ava setup.test.ts + go test -v setup_test.go } function run_tests { @@ -28,7 +29,14 @@ function run_tests { do test_case="scalers/${scaler}" counter=$((counter+1)) - ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & + # TODO - Remove condition after all tests have been migrated to Go. + if [[ $test_case == *_test.go ]] + then + go test -v $test_case > "${test_case}.log" 2>&1 & + else + ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & + fi + pid=$! echo "Running $test_case with pid: $pid" pids+=($pid) @@ -61,7 +69,14 @@ function run_tests { for test_case in "${retry_lookup[@]}" do counter=$((counter+1)) - ./node_modules/.bin/ava $test_case > "${test_case}.retry.log" 2>&1 & + # TODO - Remove condition after all tests have been migrated to Go. + if [[ $test_case == *_test.go ]] + then + go test -v $test_case > "${test_case}.retry.log" 2>&1 & + else + ./node_modules/.bin/ava $test_case > "${test_case}.retry.log" 2>&1 & + fi + pid=$! echo "Rerunning $test_case with pid: $pid" pids+=($pid) @@ -92,7 +107,7 @@ function wait_for_jobs { } function print_logs { - for test_log in $(find scalers -name "*.log") + for test_log in $(find . -name "*.log") do echo ">>> $test_log <<<" cat $test_log @@ -112,7 +127,7 @@ function print_logs { } function run_cleanup { - ./node_modules/.bin/ava cleanup.test.ts + go test -v cleanup_test.go } function print_failed { From 3da8d152494424db74e2f18b3283dd48760c53e2 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 8 Jun 2022 13:49:09 +0530 Subject: [PATCH 03/15] Sort environment variables in .env file. Signed-off-by: Vighnesh Shenoy --- tests/.env | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/.env b/tests/.env index 1954f97b1d6..8fdb14f999a 100644 --- a/tests/.env +++ b/tests/.env @@ -1,22 +1,22 @@ -AZURE_SP_APP_ID= -AZURE_SP_OBJECT_ID= -AZURE_SP_KEY= -AZURE_SP_TENANT= -AZURE_SUBSCRIPTION= -AZURE_RESOURCE_GROUP= -AZURE_LOG_ANALYTICS_WORKSPACE_ID= -AZURE_STORAGE_CONNECTION_STRING= -OPENSTACK_AUTH_URL= -OPENSTACK_USER_ID= -OPENSTACK_PASSWORD= -OPENSTACK_PROJECT_ID= -# OPENSTACK_SWIFT_URL= # OPENSTACK_REGION_NAME= +# OPENSTACK_SWIFT_URL= AWS_ACCESS_KEY= AWS_SECRET_KEY= +AZURE_DEVOPS_BUILD_DEFINITON_ID= AZURE_DEVOPS_ORGANIZATION_URL= AZURE_DEVOPS_PAT= -AZURE_DEVOPS_PROJECT= -AZURE_DEVOPS_BUILD_DEFINITON_ID= AZURE_DEVOPS_POOL_NAME= +AZURE_DEVOPS_PROJECT= AZURE_KEYVAULT_URI= +AZURE_LOG_ANALYTICS_WORKSPACE_ID= +AZURE_RESOURCE_GROUP= +AZURE_SP_APP_ID= +AZURE_SP_KEY= +AZURE_SP_OBJECT_ID= +AZURE_SP_TENANT= +AZURE_STORAGE_CONNECTION_STRING= +AZURE_SUBSCRIPTION= +OPENSTACK_AUTH_URL= +OPENSTACK_PASSWORD= +OPENSTACK_PROJECT_ID= +OPENSTACK_USER_ID= From 9040d2dced933733a67c47f621755d1b36aa1c1c Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 8 Jun 2022 15:32:49 +0530 Subject: [PATCH 04/15] Use 'e2e' build tag to exclude e2e tests from GitHub workflows. Signed-off-by: Vighnesh Shenoy --- tests/run-all.sh | 8 ++++---- tests/run-arm-smoke-tests.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/run-all.sh b/tests/run-all.sh index d05ca2c4d18..d7aaac9a7a3 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -17,7 +17,7 @@ counter=0 executed_count=0 function run_setup { - go test -v setup_test.go + go test -v -tags e2e setup_test.go } function run_tests { @@ -35,7 +35,7 @@ function run_tests { # TODO - Remove condition after all tests have been migrated to Go. if [[ $test_case == *_test.go ]] then - go test -v $test_case > "${test_case}.log" 2>&1 & + go test -v -tags e2e $test_case > "${test_case}.log" 2>&1 & else ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & fi @@ -75,7 +75,7 @@ function run_tests { # TODO - Remove condition after all tests have been migrated to Go. if [[ $test_case == *_test.go ]] then - go test -v $test_case > "${test_case}.retry.log" 2>&1 & + go test -v -tags e2e $test_case > "${test_case}.retry.log" 2>&1 & else ./node_modules/.bin/ava $test_case > "${test_case}.retry.log" 2>&1 & fi @@ -131,7 +131,7 @@ function print_logs { } function run_cleanup { - go test -v cleanup_test.go + go test -v -tags e2e cleanup_test.go } function print_failed { diff --git a/tests/run-arm-smoke-tests.sh b/tests/run-arm-smoke-tests.sh index d596e3fb56f..d3c7de63df9 100755 --- a/tests/run-arm-smoke-tests.sh +++ b/tests/run-arm-smoke-tests.sh @@ -19,7 +19,7 @@ failed_lookup=() counter=0 function run_setup { - go test -v setup_test.go + go test -v -tags e2e setup_test.go } function run_tests { @@ -32,7 +32,7 @@ function run_tests { # TODO - Remove condition after all tests have been migrated to Go. if [[ $test_case == *_test.go ]] then - go test -v $test_case > "${test_case}.log" 2>&1 & + go test -v -tags e2e $test_case > "${test_case}.log" 2>&1 & else ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & fi @@ -72,7 +72,7 @@ function run_tests { # TODO - Remove condition after all tests have been migrated to Go. if [[ $test_case == *_test.go ]] then - go test -v $test_case > "${test_case}.retry.log" 2>&1 & + go test -v -tags e2e $test_case > "${test_case}.retry.log" 2>&1 & else ./node_modules/.bin/ava $test_case > "${test_case}.retry.log" 2>&1 & fi @@ -127,7 +127,7 @@ function print_logs { } function run_cleanup { - go test -v cleanup_test.go + go test -v -tags e2e cleanup_test.go } function print_failed { From 2b74968b5e47cb662ee3d02eaabc7c74b2b88c0f Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Wed, 8 Jun 2022 16:56:22 +0530 Subject: [PATCH 05/15] Add global setup and cleanup code. Signed-off-by: Vighnesh Shenoy --- go.mod | 4 +- go.sum | 4 + tests/cleanup.test.ts | 31 ------- tests/cleanup_test.go | 34 +++++++ tests/helper.go | 169 +++++++++++++++++++++++++++++++++++ tests/run-all.sh | 2 +- tests/run-arm-smoke-tests.sh | 2 +- tests/setup.test.ts | 132 --------------------------- tests/setup_test.go | 132 +++++++++++++++++++++++++++ 9 files changed, 343 insertions(+), 167 deletions(-) delete mode 100644 tests/cleanup.test.ts create mode 100644 tests/cleanup_test.go create mode 100644 tests/helper.go delete mode 100644 tests/setup.test.ts create mode 100644 tests/setup_test.go diff --git a/go.mod b/go.mod index c4ca0af0302..7cc3d4436c1 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/robfig/cron/v3 v3.0.1 github.com/streadway/amqp v1.0.0 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.7.2 github.com/tidwall/gjson v1.14.1 github.com/xdg/scram v1.0.5 github.com/xhit/go-str2duration/v2 v2.0.0 @@ -263,7 +263,7 @@ require ( gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.23.5 // indirect k8s.io/component-base v0.23.6 // indirect k8s.io/gengo v0.0.0-20220307231824-4627b89bbf1b // indirect diff --git a/go.sum b/go.sum index a1e6db32933..bb68e0f60f1 100644 --- a/go.sum +++ b/go.sum @@ -968,6 +968,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -1537,6 +1539,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts deleted file mode 100644 index a991ab6bfd5..00000000000 --- a/tests/cleanup.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import * as sh from 'shelljs' -import test from 'ava' - -const workloadIdentityNamespace = "azure-workload-identity-system" -const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TESTS'] - -test.before('setup shelljs', () => { - sh.config.silent = true -}) - -test.serial('Remove KEDA', t => { - let result = sh.exec('(cd .. && make undeploy)') - if (result.code !== 0) { - t.fail('error removing keda. ' + result) - } - t.pass('KEDA undeployed successfully using make undeploy command') -}) - -test.serial('remove azure workload identity kubernetes components', t => { - if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { - t.pass('skipping as workload identity tests are disabled') - return - } - - t.is(0, - sh.exec(`helm uninstall workload-identity-webhook --namespace ${workloadIdentityNamespace}`).code, - 'should be able to uninstall workload identity webhook' - ) - - sh.exec(`kubectl delete ns ${workloadIdentityNamespace}`) -}) diff --git a/tests/cleanup_test.go b/tests/cleanup_test.go new file mode 100644 index 00000000000..6570dc30bfd --- /dev/null +++ b/tests/cleanup_test.go @@ -0,0 +1,34 @@ +//go:build e2e +// +build e2e + +package tests + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + . "github.com/kedacore/keda/v2/tests" +) + +func TestRemoveKEDA(t *testing.T) { + out, err := ExecuteCommandWithDir("make undeploy", "..") + require.NoErrorf(t, err, "error removing KEDA - %s", err) + + t.Log(string(out)) + t.Log("KEDA removed successfully using 'make undeploy' command") +} + +func TestRemoveWorkloadIdentityComponents(t *testing.T) { + if AzureRunWorkloadIdentityTests == "" || AzureRunWorkloadIdentityTests == "false" { + t.Skip("skipping as workload identity tests are disabled") + } + + _, err := ExecuteCommand(fmt.Sprintf("helm uninstall workload-identity-webhook --namespace %s", AzureWorkloadIdentityNamespace)) + require.NoErrorf(t, err, "cannot uninstall workload identity webhook - %s", err) + + Kc = GetKubernetesClient(t) + + DeleteNamespace(t, Kc, AzureWorkloadIdentityNamespace) +} diff --git a/tests/helper.go b/tests/helper.go new file mode 100644 index 00000000000..fc2f8f9c73b --- /dev/null +++ b/tests/helper.go @@ -0,0 +1,169 @@ +package tests + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "testing" + "text/template" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +const ( + AzureWorkloadIdentityNamespace = "azure-workload-identity-system" + KEDANamespace = "keda" + KEDAOperator = "keda-operator" + KEDAMetricsAPIServer = "keda-metrics-apiserver" +) + +// Env variables assertd for setup and cleanup. +var ( + AzureADTenantID = os.Getenv("AZURE_SP_TENANT") + AzureRunWorkloadIdentityTests = os.Getenv("AZURE_RUN_WORKLOAD_IDENTITY_TESTS") +) + +var ( + Kc *kubernetes.Clientset +) + +type ExecutionError struct { + StdError []byte +} + +func (ee ExecutionError) Error() string { + return string(ee.StdError) +} + +func ParseCommand(cmdWithArgs string) *exec.Cmd { + splitCmd := strings.Fields(cmdWithArgs) + + return exec.Command(splitCmd[0], splitCmd[1:]...) +} + +func ParseCommandWithDir(cmdWithArgs, dir string) *exec.Cmd { + cmd := ParseCommand(cmdWithArgs) + cmd.Dir = dir + + return cmd +} + +func ExecuteCommand(cmdWithArgs string) ([]byte, error) { + out, err := ParseCommand(cmdWithArgs).Output() + if err != nil { + exitError, ok := err.(*exec.ExitError) + if ok { + return out, ExecutionError{StdError: exitError.Stderr} + } + } + + return out, err +} + +func ExecuteCommandWithDir(cmdWithArgs, dir string) ([]byte, error) { + out, err := ParseCommandWithDir(cmdWithArgs, dir).Output() + if err != nil { + exitError, ok := err.(*exec.ExitError) + if ok { + return out, ExecutionError{StdError: exitError.Stderr} + } + } + + return out, err +} + +func GetKubernetesClient(t *testing.T) *kubernetes.Clientset { + if Kc != nil { + return Kc + } + + kubeConfig, err := config.GetConfig() + assert.NoErrorf(t, err, "cannot fetch kube config file - %s", err) + + Kc, err = kubernetes.NewForConfig(kubeConfig) + assert.NoErrorf(t, err, "cannot create kubernetes client - %s", err) + + return Kc +} + +func CreateNamespace(t *testing.T, kc *kubernetes.Clientset, nsName string) { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + Labels: map[string]string{"type": "e2e"}, + }, + } + + _, err := kc.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) + assert.NoErrorf(t, err, "cannot create kubernetes namespace - %s", err) +} + +func DeleteNamespace(t *testing.T, kc *kubernetes.Clientset, nsName string) { + err := Kc.CoreV1().Namespaces().Delete(context.Background(), nsName, metav1.DeleteOptions{}) + assert.NoErrorf(t, err, "cannot delete kubernetes namespace - %s", err) +} + +func WaitForDeploymentReplicaCount(t *testing.T, kc *kubernetes.Clientset, name, namespace string, + target, iterations, intervalSeconds int) bool { + for i := 0; i < iterations; i++ { + deployment, _ := kc.AppsV1().Deployments(namespace).Get(context.Background(), name, metav1.GetOptions{}) + replicas := deployment.Status.Replicas + + t.Logf("Waiting for deployment replicas to hit target. Deployment - %s, Current - %d, Target - %d", + name, replicas, target) + + if replicas == int32(target) { + return true + } + + time.Sleep(time.Duration(intervalSeconds) * time.Second) + } + + return false +} + +func KubectlApplyWithTemplate(t *testing.T, config string, data interface{}) { + tmpl, err := template.New("kubernetes resource template").Parse(config) + assert.NoErrorf(t, err, "cannot parse template - %s", err) + + tempFile, err := ioutil.TempFile("", "tempTemplateFile") + assert.NoErrorf(t, err, "cannot create temp file - %s", err) + + defer os.Remove(tempFile.Name()) + + err = tmpl.Execute(tempFile, data) + assert.NoErrorf(t, err, "cannot insert data into template - %s", err) + + _, err = ExecuteCommand(fmt.Sprintf("kubectl apply -f %s", tempFile.Name())) + assert.NoErrorf(t, err, "cannot apply file - %s", err) + + err = tempFile.Close() + assert.NoErrorf(t, err, "cannot close temp file - %s", err) +} + +func KubectlDeleteWithTemplate(t *testing.T, config string, data interface{}) { + tmpl, err := template.New("kubernetes resource template").Parse(config) + assert.NoErrorf(t, err, "cannot parse template - %s", err) + + tempFile, err := ioutil.TempFile("", "tempTemplateFile") + assert.NoErrorf(t, err, "cannot create temp file - %s", err) + + defer os.Remove(tempFile.Name()) + + err = tmpl.Execute(tempFile, data) + assert.NoErrorf(t, err, "cannot insert data into template - %s", err) + + _, err = ExecuteCommand(fmt.Sprintf("kubectl delete -f %s", tempFile.Name())) + assert.NoErrorf(t, err, "cannot apply file - %s", err) + + err = tempFile.Close() + assert.NoErrorf(t, err, "cannot close temp file - %s", err) +} diff --git a/tests/run-all.sh b/tests/run-all.sh index d7aaac9a7a3..8f3a0f0fab2 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -eu +set -u # TODO - Remove TypeScript regex after all tests have been migrated to Go. E2E_REGEX_GO="${E2E_TEST_REGEX}_*test.go" diff --git a/tests/run-arm-smoke-tests.sh b/tests/run-arm-smoke-tests.sh index d3c7de63df9..a0a1bc65fc1 100755 --- a/tests/run-arm-smoke-tests.sh +++ b/tests/run-arm-smoke-tests.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -eu +set -u DIR=$(dirname "$0") cd $DIR diff --git a/tests/setup.test.ts b/tests/setup.test.ts deleted file mode 100644 index 5de723e2415..00000000000 --- a/tests/setup.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import * as sh from 'shelljs' -import * as k8s from '@kubernetes/client-node' -import test from 'ava' - -const kc = new k8s.KubeConfig() -kc.loadFromDefault() - -const AZURE_AD_TENANT_ID = process.env['AZURE_SP_TENANT'] -const RUN_WORKLOAD_IDENTITY_TESTS = process.env['AZURE_RUN_WORKLOAD_IDENTITY_TESTS'] -const workloadIdentityNamespace = "azure-workload-identity-system" - -test.before('configure shelljs', () => { - sh.config.silent = true -}) - -test.serial('Verify all commands', t => { - for (const command of ['kubectl']) { - if (!sh.which(command)) { - t.fail(`${command} is required for setup`) - } - } - t.pass() -}) - -test.serial('Verify environment variables', t => { - const cluster = kc.getCurrentCluster() - t.truthy(cluster, 'Make sure kubectl is logged into a cluster.') -}) - -test.serial('Get Kubernetes version', t => { - let result = sh.exec('kubectl version ') - if (result.code !== 0) { - t.fail('error getting Kubernetes version') - } else { - t.log('kubernetes version: ' + result.stdout) - t.pass() - } -}) - -test.serial('setup helm', t => { - // check if helm is already installed. - let result = sh.exec('helm version') - if(result.code == 0) { - t.pass('helm is already installed. skipping setup') - return - } - t.is(0, sh.exec(`curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3`).code, 'should be able to download helm script') - t.is(0, sh.exec(`chmod 700 get_helm.sh`).code, 'should be able to change helm script permissions') - t.is(0, sh.exec(`./get_helm.sh`).code, 'should be able to download helm') - t.is(0, sh.exec(`helm version`).code, 'should be able to get helm version') -}) - -test.serial('setup and verify azure workload identity kubernetes components', t => { - if (!RUN_WORKLOAD_IDENTITY_TESTS || RUN_WORKLOAD_IDENTITY_TESTS == 'false') { - t.pass('skipping as workload identity tests are disabled') - return - } - - // check if helm is already installed. - let result = sh.exec('helm version') - if (result.code != 0) { - t.fail('helm is not installed') - return - } - - // Add Azure AD Workload Identity Helm Repo - t.is(0, - sh.exec('helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts').code, - 'should be able to add Azure AD workload identity helm repo' - ) - t.is(0, - sh.exec(`helm repo update azure-workload-identity`).code, - "should be able to update" - ) - - // Install Workload Identity Webhook if not present - t.is(0, - sh.exec(`helm upgrade --install workload-identity-webhook azure-workload-identity/workload-identity-webhook --namespace ${workloadIdentityNamespace} --create-namespace --set azureTenantID="${AZURE_AD_TENANT_ID}"`).code, - 'should be able to install workload identity webhook' - ) - - let success = false - for (let i = 0; i < 20; i++) { - result = sh.exec( - `kubectl get deployment.apps/azure-wi-webhook-controller-manager -n ${workloadIdentityNamespace} -o jsonpath="{.status.readyReplicas}"` - ) - const parsedPods = parseInt(result.stdout, 10) - if (isNaN(parsedPods) || parsedPods != 2) { - t.log('Workload Identity webhook is not ready. sleeping') - sh.exec('sleep 5s') - } else if (parsedPods == 2) { - t.log('Workload Identity webhook is ready') - success = true - sh.exec('sleep 120s') // Sleep for some time for webhook to setup properly - break - } - } - - t.true(success, 'expected workload identity deployments to start 2 pods successfully') -}) - -test.serial('Deploy KEDA', t => { - let result = sh.exec('(cd .. && make deploy)') - if (result.code !== 0) { - t.fail('error deploying keda. ' + result) - } - t.pass('KEDA deployed successfully using make deploy command') -}) - -test.serial('verifyKeda', t => { - let success = false - for (let i = 0; i < 20; i++) { - let resultOperator = sh.exec( - 'kubectl get deployment.apps/keda-operator --namespace keda -o jsonpath="{.status.readyReplicas}"' - ) - let resultMetrics = sh.exec( - 'kubectl get deployment.apps/keda-metrics-apiserver --namespace keda -o jsonpath="{.status.readyReplicas}"' - ) - const parsedOperator = parseInt(resultOperator.stdout, 10) - const parsedMetrics = parseInt(resultMetrics.stdout, 10) - if (isNaN(parsedOperator) || parsedOperator != 1 || isNaN(parsedMetrics) || parsedMetrics != 1) { - t.log(`KEDA is not ready. sleeping`) - sh.exec('sleep 5s') - } else if (parsedOperator == 1 && parsedMetrics == 1) { - t.log('keda is running 1 pod for operator and 1 pod for metrics server') - success = true - break - } - } - - t.true(success, 'expected keda deployments to start 2 pods successfully') -}) diff --git a/tests/setup_test.go b/tests/setup_test.go new file mode 100644 index 00000000000..f70e6e3c1f9 --- /dev/null +++ b/tests/setup_test.go @@ -0,0 +1,132 @@ +//go:build e2e +// +build e2e + +package tests + +import ( + "context" + "fmt" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/require" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/kedacore/keda/v2/tests" +) + +func TestVerifyCommands(t *testing.T) { + commands := []string{"kubectl"} + for _, cmd := range commands { + _, err := exec.LookPath(cmd) + require.NoErrorf(t, err, "%s is required for setup - %s", cmd, err) + } +} + +func TestKubernetesConnection(t *testing.T) { + Kc = GetKubernetesClient(t) +} + +func TestKubernetesVersion(t *testing.T) { + out, err := ExecuteCommand("kubectl version") + require.NoErrorf(t, err, "error getting kubernetes version - %s", err) + + t.Logf("kubernetes version: %s", string(out)) +} + +func TestSetupHelm(t *testing.T) { + _, err := exec.LookPath("helm") + if err == nil { + t.Skip("helm is already installed. skipping setup.") + } + + _, err = ExecuteCommand("curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3") + require.NoErrorf(t, err, "cannot download helm installation shell script - %s", err) + + _, err = ExecuteCommand("chmod 700 get_helm.sh") + require.NoErrorf(t, err, "cannot change permissions for helm installation script - %s", err) + + _, err = ExecuteCommand("./get_helm.sh") + require.NoErrorf(t, err, "cannot downlaod helm - %s", err) + + _, err = ExecuteCommand("helm version") + require.NoErrorf(t, err, "cannot get helm version - %s", err) +} + +func TestSetupWorkloadIdentityComponents(t *testing.T) { + if AzureRunWorkloadIdentityTests == "" || AzureRunWorkloadIdentityTests == "false" { + t.Skip("skipping as workload identity tests are disabled") + } + + _, err := ExecuteCommand("helm version") + require.NoErrorf(t, err, "helm is not installed - %s", err) + + _, err = ExecuteCommand("helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts") + require.NoErrorf(t, err, "cannot add workload identity helm repo - %s", err) + + _, err = ExecuteCommand("helm repo update azure-workload-identity") + require.NoErrorf(t, err, "cannot update workload identity helm repo - %s", err) + + Kc = GetKubernetesClient(t) + CreateNamespace(t, Kc, AzureWorkloadIdentityNamespace) + + _, err = ExecuteCommand(fmt.Sprintf("helm upgrade --install workload-identity-webhook azure-workload-identity/workload-identity-webhook --namespace %s --set azureTenantID=%s", + AzureWorkloadIdentityNamespace, AzureADTenantID)) + require.NoErrorf(t, err, "cannot install workload identity webhook - %s", err) + + workloadIdentityDeploymentName := "azure-wi-webhook-controller-manager" + success := false + for i := 0; i < 20; i++ { + deployment, err := Kc.AppsV1().Deployments(AzureWorkloadIdentityNamespace).Get(context.Background(), workloadIdentityDeploymentName, v1.GetOptions{}) + require.NoErrorf(t, err, "unable to get workload identity webhook deployment - %s", err) + + readyReplicas := deployment.Status.ReadyReplicas + if readyReplicas != 2 { + t.Log("workload identity webhook is not ready. sleeping") + time.Sleep(5 * time.Second) + } else { + t.Log("workload identity webhook is ready") + success = true + + time.Sleep(2 * time.Minute) // sleep for some time for webhook to setup properly + break + } + } + + require.True(t, success, "expected workload identity webhook deployment to start 2 pods successfully") +} + +func TestDeployKEDA(t *testing.T) { + out, err := ExecuteCommandWithDir("make deploy", "..") + require.NoErrorf(t, err, "error deploying KEDA - %s", err) + + t.Log(string(out)) + t.Log("KEDA deployed successfully using 'make deploy' command") +} + +func TestVerifyKEDA(t *testing.T) { + success := false + for i := 0; i < 20; i++ { + operatorDeployment, err := Kc.AppsV1().Deployments(KEDANamespace).Get(context.Background(), KEDAOperator, v1.GetOptions{}) + require.NoErrorf(t, err, "unable to get %s deployment - %s", KEDAOperator, err) + + metricsServerDeployment, err := Kc.AppsV1().Deployments(KEDANamespace).Get(context.Background(), KEDAMetricsAPIServer, v1.GetOptions{}) + require.NoErrorf(t, err, "unable to get %s deployment - %s", KEDAMetricsAPIServer, err) + + operatorReadyReplicas := operatorDeployment.Status.ReadyReplicas + metricsServerReadyReplicas := metricsServerDeployment.Status.ReadyReplicas + + if operatorReadyReplicas != 1 || metricsServerReadyReplicas != 1 { + t.Log("KEDA is not ready. sleeping") + time.Sleep(5 * time.Second) + } else { + t.Logf("KEDA is running 1 pod for %s and 1 pod for %s", KEDAOperator, KEDAMetricsAPIServer) + success = true + + break + } + } + + require.True(t, success, "expected KEDA deployments to start 2 pods successfully") +} From b79aa09e857f88febefb9221c45f6beeb3f5ef9e Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 00:40:07 +0530 Subject: [PATCH 06/15] Update CHANGELOG. Signed-off-by: Vighnesh Shenoy --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6236edf7dec..2602009f2be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,8 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md ### New -- **General:** Support for Azure AD Workload Identity as a pod identity provider. ([2487](https://github.com/kedacore/keda/issues/2487)) +- **General:** Support for Azure AD Workload Identity as a pod identity provider. ([#2487](https://github.com/kedacore/keda/issues/2487)) +- **General:** Basic setup for migrating e2e tests to Go. ([#2737](https://github.com/kedacore/keda/issues/2737)) ### Improvements From f23ff3e28a3e3e3e118317bb0ee864a06fb60434 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 00:49:50 +0530 Subject: [PATCH 07/15] Add Azure Service Bus (Queue + Topic), Azure Queue Storage tests. Signed-off-by: Vighnesh Shenoy --- go.mod | 1 + go.sum | 2 + tests/helper.go | 21 +- tests/scalers/azure-queue.test.ts | 136 ---------- tests/scalers/azure-service-bus-queue.test.ts | 209 --------------- tests/scalers/azure-service-bus-topic.test.ts | 221 ---------------- .../azure_queue/azure_queue_test.go | 236 +++++++++++++++++ .../azure_service_bus_queue_test.go | 222 ++++++++++++++++ .../azure_service_bus_topic_test.go | 237 ++++++++++++++++++ 9 files changed, 717 insertions(+), 568 deletions(-) delete mode 100644 tests/scalers/azure-queue.test.ts delete mode 100644 tests/scalers/azure-service-bus-queue.test.ts delete mode 100644 tests/scalers/azure-service-bus-topic.test.ts create mode 100644 tests/scalers_go/azure_queue/azure_queue_test.go create mode 100644 tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go create mode 100644 tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go diff --git a/go.mod b/go.mod index 7cc3d4436c1..324eebfd3c7 100644 --- a/go.mod +++ b/go.mod @@ -182,6 +182,7 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/joho/godotenv v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index bb68e0f60f1..1e7c6cec855 100644 --- a/go.sum +++ b/go.sum @@ -682,6 +682,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= diff --git a/tests/helper.go b/tests/helper.go index fc2f8f9c73b..ce29ae873e9 100644 --- a/tests/helper.go +++ b/tests/helper.go @@ -11,6 +11,7 @@ import ( "text/template" "time" + "github.com/joho/godotenv" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,8 +24,12 @@ const ( KEDANamespace = "keda" KEDAOperator = "keda-operator" KEDAMetricsAPIServer = "keda-metrics-apiserver" + + DefaultHTTPTimeOut = 3000 ) +var _ = godotenv.Load() + // Env variables assertd for setup and cleanup. var ( AzureADTenantID = os.Getenv("AZURE_SP_TENANT") @@ -130,7 +135,7 @@ func WaitForDeploymentReplicaCount(t *testing.T, kc *kubernetes.Clientset, name, return false } -func KubectlApplyWithTemplate(t *testing.T, config string, data interface{}) { +func KubectlApplyWithTemplate(t *testing.T, data interface{}, config string) { tmpl, err := template.New("kubernetes resource template").Parse(config) assert.NoErrorf(t, err, "cannot parse template - %s", err) @@ -149,7 +154,13 @@ func KubectlApplyWithTemplate(t *testing.T, config string, data interface{}) { assert.NoErrorf(t, err, "cannot close temp file - %s", err) } -func KubectlDeleteWithTemplate(t *testing.T, config string, data interface{}) { +func KubectlApplyMultipleWithTemplate(t *testing.T, data interface{}, configs ...string) { + for _, config := range configs { + KubectlApplyWithTemplate(t, data, config) + } +} + +func KubectlDeleteWithTemplate(t *testing.T, data interface{}, config string) { tmpl, err := template.New("kubernetes resource template").Parse(config) assert.NoErrorf(t, err, "cannot parse template - %s", err) @@ -167,3 +178,9 @@ func KubectlDeleteWithTemplate(t *testing.T, config string, data interface{}) { err = tempFile.Close() assert.NoErrorf(t, err, "cannot close temp file - %s", err) } + +func KubectlDeleteMultipleWithTemplate(t *testing.T, data interface{}, configs ...string) { + for _, config := range configs { + KubectlDeleteWithTemplate(t, data, config) + } +} diff --git a/tests/scalers/azure-queue.test.ts b/tests/scalers/azure-queue.test.ts deleted file mode 100644 index 7365cd63f52..00000000000 --- a/tests/scalers/azure-queue.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import * as async from 'async' -import * as azure from 'azure-storage' -import * as fs from 'fs' -import * as sh from 'shelljs' -import * as tmp from 'tmp' -import test from 'ava' -import {createNamespace, waitForDeploymentReplicaCount} from "./helpers"; - -const defaultNamespace = 'azure-queue-test' -const connectionString = process.env['AZURE_STORAGE_CONNECTION_STRING'] -const queueName = 'queue-single-name' - -test.before(async t => { - if (!connectionString) { - t.fail('AZURE_STORAGE_CONNECTION_STRING environment variable is required for queue tests') - } - - const createQueueAsync = () => new Promise((resolve, _) => { - const queueSvc = azure.createQueueService(connectionString) - queueSvc.messageEncoder = new azure.QueueMessageEncoder.TextBase64QueueMessageEncoder() - queueSvc.createQueueIfNotExists(queueName, _ => { - resolve(undefined); - }) - }) - await createQueueAsync() - - sh.config.silent = true - const base64ConStr = Buffer.from(connectionString).toString('base64') - const tmpFile = tmp.fileSync() - fs.writeFileSync(tmpFile.name, deployYaml.replace('{{CONNECTION_STRING_BASE64}}', base64ConStr)) - createNamespace(defaultNamespace) - t.is( - 0, - sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${defaultNamespace}`).code, - 'creating a deployment should work.' - ) - t.true(await waitForDeploymentReplicaCount(0, 'test-deployment', defaultNamespace, 60, 1000), 'replica count should be 0 after 1 minute') -}) - -test.serial( - 'Deployment should scale to 4 with 10,000 messages on the queue then back to 0', - async t => { - const queueSvc = azure.createQueueService(connectionString) - queueSvc.messageEncoder = new azure.QueueMessageEncoder.TextBase64QueueMessageEncoder() - await async.mapLimit( - Array(1000).keys(), - 20, - (n, cb) => queueSvc.createMessage(queueName, `test ${n}`, cb) - ) - - // Scaling out when messages available - t.true(await waitForDeploymentReplicaCount(1, 'test-deployment', defaultNamespace, 60, 1000), 'replica count should be 1 after 1 minutes') - - queueSvc.clearMessages(queueName, _ => {}) - - // Scaling in when no available messages - t.true(await waitForDeploymentReplicaCount(0, 'test-deployment', defaultNamespace, 300, 1000), 'replica count should be 0 after 5 minute') - } -) - -test.after.always.cb('clean up azure-queue deployment', t => { - const resources = [ - 'scaledobject.keda.sh/test-scaledobject', - 'secret/test-secrets', - 'deployment.apps/test-deployment', - ] - - for (const resource of resources) { - sh.exec(`kubectl delete ${resource} --namespace ${defaultNamespace}`) - } - sh.exec(`kubectl delete namespace ${defaultNamespace}`) - - // delete test queue - const queueSvc = azure.createQueueService(connectionString) - queueSvc.deleteQueueIfExists(queueName, err => { - t.falsy(err, 'should delete test queue successfully') - t.end() - }) -}) - -const deployYaml = `apiVersion: v1 -kind: Secret -metadata: - name: test-secrets - labels: -data: - AzureWebJobsStorage: {{CONNECTION_STRING_BASE64}} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: test-deployment - labels: - app: test-deployment -spec: - replicas: 0 - selector: - matchLabels: - app: test-deployment - template: - metadata: - name: - namespace: - labels: - app: test-deployment - spec: - containers: - - name: test-deployment - image: ghcr.io/kedacore/tests-azure-queue - resources: - ports: - env: - - name: FUNCTIONS_WORKER_RUNTIME - value: node - - name: AzureWebJobsStorage - valueFrom: - secretKeyRef: - name: test-secrets - key: AzureWebJobsStorage ---- -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: test-scaledobject -spec: - scaleTargetRef: - name: test-deployment - pollingInterval: 5 - minReplicaCount: 0 - maxReplicaCount: 1 - cooldownPeriod: 10 - triggers: - - type: azure-queue - metadata: - queueName: ${queueName} - connectionFromEnv: AzureWebJobsStorage` diff --git a/tests/scalers/azure-service-bus-queue.test.ts b/tests/scalers/azure-service-bus-queue.test.ts deleted file mode 100644 index e3fdbe60349..00000000000 --- a/tests/scalers/azure-service-bus-queue.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -import * as sh from "shelljs" -import * as azure from "@azure/service-bus" -import test from "ava" -import { createNamespace, createYamlFile, waitForDeploymentReplicaCount } from "./helpers" - -const connectionString = process.env["AZURE_SERVICE_BUS_CONNECTION_STRING"] -const queueName = "sb-queue" - -const testName = "test-azure-service-bus-queue" -const testNamespace = `${testName}-ns` -const secretName = `${testName}-secret` -const deploymentName = `${testName}-deployment` -const triggerAuthName = `${testName}-trigger-auth` -const scaledObjectName = `${testName}-scaled-object` - -test.before(async t => { - if (!connectionString) { - t.fail("AZURE_SERVICE_BUS_CONNECTION_STRING environment variable is required for service bus tests") - } - - sh.config.silent = true - - // Create queue within the Service Bus Namespace - const serviceBusAdminClient = new azure.ServiceBusAdministrationClient(connectionString) - const queueExists = await serviceBusAdminClient.queueExists(queueName) - // Clean up (delete) queue if already exists and create again - if (queueExists) { - await serviceBusAdminClient.deleteQueue(queueName) - } - await serviceBusAdminClient.createQueue(queueName) - - // Create Kubernetes Namespace - createNamespace(testNamespace) - - // Create Secret - const base64ConStr = Buffer.from(connectionString).toString("base64") - const secretFileName = createYamlFile(secretYaml.replace("{{CONNECTION}}", base64ConStr)) - - t.is( - sh.exec(`kubectl apply -f ${secretFileName} -n ${testNamespace}`).code, - 0, - "Creating a secret should work" - ) - - // Create deployment - t.is( - sh.exec(`kubectl apply -f ${createYamlFile(deploymentYaml)} -n ${testNamespace}`).code, - 0, - "Creating a deployment should work" - ) - - // Create trigger auth resource - t.is( - sh.exec(`kubectl apply -f ${createYamlFile(triggerAuthYaml)} -n ${testNamespace}`).code, - 0, - "Creating a trigger authentication resource should work" - ) - - // Create scaled object - t.is( - sh.exec(`kubectl apply -f ${createYamlFile(scaledObjectYaml)} -n ${testNamespace}`).code, - 0, - "Creating a scaled object should work" - ) - - t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 60, 1000), "Replica count should be 0 after 1 minute") -}) - -test.serial("Deployment should scale up with messages on service bus queue", async t => { - // Send messages to service bus queue - const serviceBusClient = new azure.ServiceBusClient(connectionString) - const sender = serviceBusClient.createSender(queueName) - - const messages: azure.ServiceBusMessage[] = [ - {"body": "1"}, - {"body": "2"}, - {"body": "3"}, - {"body": "4"}, - {"body": "5"}, - ] - - await sender.sendMessages(messages) - - await serviceBusClient.close() - - // Scale out when messages available - t.true(await waitForDeploymentReplicaCount(1, deploymentName, testNamespace, 60, 1000), "Replica count should be 1 after 1 minute") -}) - -test.serial("Deployment should scale down with messages on service bus queue", async t => { - // Receive messages from service bus queue - const serviceBusClient = new azure.ServiceBusClient(connectionString) - const receiver = serviceBusClient.createReceiver(queueName) - - var numOfReceivedMessages = 0 - - while (numOfReceivedMessages < 5) { - const messages = await receiver.receiveMessages(10, { - maxWaitTimeInMs: 60 * 1000, - }) - - for (const message of messages) { - await receiver.completeMessage(message) - numOfReceivedMessages += 1 - } - } - - await serviceBusClient.close() - - // Scale down when messages unavailable - t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 60, 1000), "Replica count should be 0 after 1 minute") -}) - -test.after.always("Clean up E2E K8s objects", async t => { - const resources = [ - `scaledobject.keda.sh/${scaledObjectName}`, - `triggerauthentications.keda.sh/${triggerAuthName}`, - `deployments.apps/${deploymentName}`, - `secrets/${secretName}`, - ] - - for (const resource of resources) { - sh.exec(`kubectl delete ${resource} -n ${testNamespace}`) - } - - sh.exec(`kubectl delete ns ${testNamespace}`) - - // Delete queue - const serviceBusAdminClient = new azure.ServiceBusAdministrationClient(connectionString) - const response = await serviceBusAdminClient.deleteQueue(queueName) - t.is( - response._response.status, - 200, - "Queue deletion must succeed" - ) -}) - -// YAML Definitions for Kubernetes resources -// Secret -const secretYaml = -`apiVersion: v1 -kind: Secret -metadata: - name: ${secretName} - namespace: ${testNamespace} -type: Opaque -data: - connection: {{CONNECTION}} -` - -// Deployment -const deploymentYaml = -`apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${deploymentName} - namespace: ${testNamespace} -spec: - replicas: 0 - selector: - matchLabels: - app: ${deploymentName} - template: - metadata: - labels: - app: ${deploymentName} - spec: - containers: - - name: nginx - image: nginx:1.16.1 -` - -// Trigger Authentication -const triggerAuthYaml = -`apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: ${triggerAuthName} - namespace: ${testNamespace} -spec: - secretTargetRef: - - parameter: connection - name: ${secretName} - key: connection -` - -// Scaled Object -const scaledObjectYaml = -`apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: ${scaledObjectName} - namespace: ${testNamespace} - labels: - deploymentName: ${deploymentName} -spec: - scaleTargetRef: - name: ${deploymentName} - pollingInterval: 5 - cooldownPeriod: 10 - minReplicaCount: 0 - maxReplicaCount: 1 - triggers: - - type: azure-servicebus - metadata: - queueName: ${queueName} - authenticationRef: - name: ${triggerAuthName} -` diff --git a/tests/scalers/azure-service-bus-topic.test.ts b/tests/scalers/azure-service-bus-topic.test.ts deleted file mode 100644 index 00596586d9b..00000000000 --- a/tests/scalers/azure-service-bus-topic.test.ts +++ /dev/null @@ -1,221 +0,0 @@ -import * as sh from "shelljs" -import * as azure from "@azure/service-bus" -import test from "ava" -import { createNamespace, createYamlFile, waitForDeploymentReplicaCount } from "./helpers" - -const connectionString = process.env["AZURE_SERVICE_BUS_CONNECTION_STRING"] -const topicName = "sb-topic" -const subscriptionName = "sb-subscription" - -const testName = "test-azure-service-bus-topic" -const testNamespace = `${testName}-ns` -const secretName = `${testName}-secret` -const deploymentName = `${testName}-deployment` -const triggerAuthName = `${testName}-trigger-auth` -const scaledObjectName = `${testName}-scaled-object` - -test.before(async t => { - if (!connectionString) { - t.fail("AZURE_SERVICE_BUS_CONNECTION_STRING environment variable is required for service bus tests") - } - - sh.config.silent = true - - // Create topic within the Service Bus Namespace - const serviceBusAdminClient = new azure.ServiceBusAdministrationClient(connectionString) - const topicExists = await serviceBusAdminClient.topicExists(topicName) - // Clean up (delete) topic if already exists and create again - if (topicExists) { - await serviceBusAdminClient.deleteTopic(topicName) - } - await serviceBusAdminClient.createTopic(topicName) - - // Create subscription within topic - await serviceBusAdminClient.createSubscription(topicName, subscriptionName) - - // Create Kubernetes Namespace - createNamespace(testNamespace) - - // Create Secret - const base64ConStr = Buffer.from(connectionString).toString("base64") - const secretFileName = createYamlFile(secretYaml.replace("{{CONNECTION}}", base64ConStr)) - - t.is( - sh.exec(`kubectl apply -f ${secretFileName} -n ${testNamespace}`).code, - 0, - "Creating a secret should work" - ) - - // Create deployment - t.is( - sh.exec(`kubectl apply -f ${createYamlFile(deploymentYaml)} -n ${testNamespace}`).code, - 0, - "Creating a deployment should work" - ) - - // Create trigger auth resource - t.is( - sh.exec(`kubectl apply -f ${createYamlFile(triggerAuthYaml)} -n ${testNamespace}`).code, - 0, - "Creating a trigger authentication resource should work" - ) - - // Create scaled object - t.is( - sh.exec(`kubectl apply -f ${createYamlFile(scaledObjectYaml)} -n ${testNamespace}`).code, - 0, - "Creating a scaled object should work" - ) - - t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 60, 1000), "Replica count should be 0 after 1 minute") -}) - -test.serial("Deployment should scale up with messages on service bus topic", async t => { - // Send messages to service bus topic - const serviceBusClient = new azure.ServiceBusClient(connectionString) - const sender = serviceBusClient.createSender(topicName) - - const messages: azure.ServiceBusMessage[] = [ - {"body": "1"}, - {"body": "2"}, - {"body": "3"}, - {"body": "4"}, - {"body": "5"}, - ] - - await sender.sendMessages(messages) - - await serviceBusClient.close() - - // Scale out when messages available - t.true(await waitForDeploymentReplicaCount(1, deploymentName, testNamespace, 60, 1000), "Replica count should be 1 after 1 minute") -}) - -test.serial("Deployment should scale down with messages on service bus topic", async t => { - // Receive messages from service bus topic - const serviceBusClient = new azure.ServiceBusClient(connectionString) - const receiver = serviceBusClient.createReceiver(topicName, subscriptionName) - - var numOfReceivedMessages = 0 - - while (numOfReceivedMessages < 5) { - const messages = await receiver.receiveMessages(10, { - maxWaitTimeInMs: 60 * 1000, - }) - - for (const message of messages) { - await receiver.completeMessage(message) - numOfReceivedMessages += 1 - } - } - - await serviceBusClient.close() - - // Scale down when messages unavailable - t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 60, 1000), "Replica count should be 0 after 1 minute") -}) - -test.after.always("Clean up E2E K8s objects", async t => { - const resources = [ - `scaledobject.keda.sh/${scaledObjectName}`, - `triggerauthentications.keda.sh/${triggerAuthName}`, - `deployments.apps/${deploymentName}`, - `secrets/${secretName}`, - ] - - for (const resource of resources) { - sh.exec(`kubectl delete ${resource} -n ${testNamespace}`) - } - - sh.exec(`kubectl delete ns ${testNamespace}`) - - // Delete subscription - const serviceBusAdminClient = new azure.ServiceBusAdministrationClient(connectionString) - var response = await serviceBusAdminClient.deleteSubscription(topicName, subscriptionName) - t.is( - response._response.status, - 200, - "Subscription deletion must succeed" - ) - - response = await serviceBusAdminClient.deleteTopic(topicName) - t.is( - response._response.status, - 200, - "Topic deletion must succeed" - ) -}) - -// YAML Definitions for Kubernetes resources -// Secret -const secretYaml = -`apiVersion: v1 -kind: Secret -metadata: - name: ${secretName} - namespace: ${testNamespace} -type: Opaque -data: - connection: {{CONNECTION}} -` - -// Deployment -const deploymentYaml = -`apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${deploymentName} - namespace: ${testNamespace} -spec: - replicas: 0 - selector: - matchLabels: - app: ${deploymentName} - template: - metadata: - labels: - app: ${deploymentName} - spec: - containers: - - name: nginx - image: nginx:1.16.1 -` - -// Trigger Authentication -const triggerAuthYaml = -`apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: ${triggerAuthName} - namespace: ${testNamespace} -spec: - secretTargetRef: - - parameter: connection - name: ${secretName} - key: connection -` - -// Scaled Object -const scaledObjectYaml = -`apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: ${scaledObjectName} - namespace: ${testNamespace} - labels: - deploymentName: ${deploymentName} -spec: - scaleTargetRef: - name: ${deploymentName} - pollingInterval: 5 - cooldownPeriod: 10 - minReplicaCount: 0 - maxReplicaCount: 1 - triggers: - - type: azure-servicebus - metadata: - topicName: ${topicName} - subscriptionName: ${subscriptionName} - authenticationRef: - name: ${triggerAuthName} -` diff --git a/tests/scalers_go/azure_queue/azure_queue_test.go b/tests/scalers_go/azure_queue/azure_queue_test.go new file mode 100644 index 00000000000..64121c7f1a5 --- /dev/null +++ b/tests/scalers_go/azure_queue/azure_queue_test.go @@ -0,0 +1,236 @@ +package azure_queue_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" + "github.com/joho/godotenv" + "github.com/stretchr/testify/require" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/scalers/azure" + kedautil "github.com/kedacore/keda/v2/pkg/util" + . "github.com/kedacore/keda/v2/tests" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "azure-queue-test" +) + +var ( + connectionString = os.Getenv("AZURE_STORAGE_CONNECTION_STRING") + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + queueName = fmt.Sprintf("%s-queue", testName) +) + +type templateData struct { + TestNamespace string + SecretName string + Connection string + DeploymentName string + ScaledObjectName string + QueueName string +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + AzureWebJobsStorage: {{.Connection}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: ghcr.io/kedacore/tests-azure-queue + resources: + env: + - name: FUNCTIONS_WORKER_RUNTIME + value: node + - name: AzureWebJobsStorage + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: AzureWebJobsStorage +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: 1 + cooldownPeriod: 10 + triggers: + - type: azure-queue + metadata: + queueName: {{.QueueName}} + connectionFromEnv: AzureWebJobsStorage +` +) + +func TestSetup(t *testing.T) { + require.NotEmpty(t, connectionString, "AZURE_STORAGE_CONNECTION_STRING env variable is required for service bus tests") + + // Create Queue + httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) + credential, endpoint, err := azure.ParseAzureStorageQueueConnection( + context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") + require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + + p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) + serviceURL := azqueue.NewServiceURL(*endpoint, p) + queueURL := serviceURL.NewQueueURL(queueName) + + _, err = queueURL.Create(context.Background(), azqueue.Metadata{}) + require.NoErrorf(t, err, "cannot create storage queue - %s", err) + + // Create kubernetes resources + Kc = GetKubernetesClient(t) + CreateNamespace(t, Kc, testNamespace) + + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + data := templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + QueueName: queueName, + } + + KubectlApplyMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, scaledObjectTemplate) + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") +} + +func TestScaleUp(t *testing.T) { + // Create Queue + httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) + credential, endpoint, err := azure.ParseAzureStorageQueueConnection( + context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") + require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + + p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) + serviceURL := azqueue.NewServiceURL(*endpoint, p) + queueURL := serviceURL.NewQueueURL(queueName) + + messageURL := queueURL.NewMessagesURL() + + for i := 0; i < 5; i++ { + go func(t *testing.T, idx int) { + for j := 0; j < 200; j++ { + msg := fmt.Sprintf("Routine %d - Message - %d", idx, j) + _, err := messageURL.Enqueue(context.Background(), msg, 0*time.Second, time.Hour) + require.NoErrorf(t, err, "cannot enqueue message - %s", err) + } + }(t, i) + } + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 1, 60, 1), + "replica count should be 0 after a minute") +} + +func TestScaleDown(t *testing.T) { + // Create Queue + httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) + credential, endpoint, err := azure.ParseAzureStorageQueueConnection( + context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") + require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + + p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) + serviceURL := azqueue.NewServiceURL(*endpoint, p) + queueURL := serviceURL.NewQueueURL(queueName) + + messageURL := queueURL.NewMessagesURL() + + // Clear queue + // I am not sure if this is the best way to go about this. If someone can find a better way + // please raise a PR. + for { + props, err := queueURL.GetProperties(context.Background()) + require.NoErrorf(t, err, "cannot fetch queue properties - %s", err) + + if props.ApproximateMessagesCount() > 0 { + _, err := messageURL.Clear(context.Background()) + require.NoErrorf(t, err, "cannot clear storage queue - %s", err) + } else { + break + } + } + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") +} + +func TestCleanup(t *testing.T) { + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + data := templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + QueueName: queueName, + } + + // Delete kubernetes resources + KubectlDeleteMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, scaledObjectTemplate) + + Kc = GetKubernetesClient(t) + DeleteNamespace(t, Kc, testNamespace) + + // Create Queue + httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) + credential, endpoint, err := azure.ParseAzureStorageQueueConnection( + context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") + require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + + p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) + serviceURL := azqueue.NewServiceURL(*endpoint, p) + queueURL := serviceURL.NewQueueURL(queueName) + + _, err = queueURL.Delete(context.Background()) + require.NoErrorf(t, err, "cannot create storage queue - %s", err) +} diff --git a/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go b/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go new file mode 100644 index 00000000000..25129f9d955 --- /dev/null +++ b/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go @@ -0,0 +1,222 @@ +package azure_service_bus_queue_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + "time" + + servicebus "github.com/Azure/azure-service-bus-go" + "github.com/joho/godotenv" + "github.com/stretchr/testify/require" + + . "github.com/kedacore/keda/v2/tests" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "azure-service-bus-queue-test" +) + +var ( + connectionString = os.Getenv("AZURE_SERVICE_BUS_CONNECTION_STRING") + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + queueName = fmt.Sprintf("%s-queue", testName) +) + +type templateData struct { + TestNamespace string + SecretName string + Connection string + DeploymentName string + TriggerAuthName string + ScaledObjectName string + QueueName string +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +type: Opaque +data: + connection: {{.Connection}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: nginx + image: nginx:1.16.1 +` + + triggerAuthTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: connection + name: {{.SecretName}} + key: connection +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} + labels: + deploymentName: {{.DeploymentName}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + cooldownPeriod: 10 + minReplicaCount: 0 + maxReplicaCount: 1 + triggers: + - type: azure-servicebus + metadata: + queueName: {{.QueueName}} + authenticationRef: + name: {{.TriggerAuthName}} +` +) + +func TestSetup(t *testing.T) { + require.NotEmpty(t, connectionString, "AZURE_SERVICE_BUS_CONNECTION_STRING env variable is required for service bus tests") + + // Connect to service bus namespace. + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbQueueManager := sbNamespace.NewQueueManager() + sbQueues, err := sbQueueManager.List(context.Background()) + require.NoErrorf(t, err, "cannot fetch queue list for service bus namespace - %s", err) + + // Delete service bus queue if already exists. + for _, queue := range sbQueues { + if queue.Name == queueName { + t.Log("Service Bus Queue already exists. Deleting.") + err := sbQueueManager.Delete(context.Background(), queueName) + require.NoErrorf(t, err, "cannot delete existing service bus queue - %s", err) + } + } + + // Create service bus queue. + _, err = sbQueueManager.Put(context.Background(), queueName) + require.NoErrorf(t, err, "cannot create service bus queue - %s", err) + + Kc = GetKubernetesClient(t) + CreateNamespace(t, Kc, testNamespace) + + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + data := templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + QueueName: queueName, + } + + // Create kubernetes resources + KubectlApplyMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") +} + +func TestScaleUp(t *testing.T) { + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbQueue, err := sbNamespace.NewQueue(queueName) + require.NoErrorf(t, err, "cannot create client for queue - %s", err) + + for i := 0; i < 5; i++ { + _ = sbQueue.Send(context.Background(), servicebus.NewMessageFromString(fmt.Sprintf("Message - %d", i))) + } + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 1, 60, 1), + "replica count should be 1 after 1 minute") +} + +func TestScaleDown(t *testing.T) { + var messageHandlerFunc servicebus.HandlerFunc = func(ctx context.Context, msg *servicebus.Message) error { + return msg.Complete(ctx) + } + + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbQueue, err := sbNamespace.NewQueue(queueName) + require.NoErrorf(t, err, "cannot create client for queue - %s", err) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _ = sbQueue.Receive(ctx, messageHandlerFunc) + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after 1 minute") +} + +func TestCleanup(t *testing.T) { + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + data := templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + QueueName: queueName, + } + + // Delete kubernetes resources + KubectlDeleteMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) + + Kc = GetKubernetesClient(t) + DeleteNamespace(t, Kc, testNamespace) + + // Delete service bus queue. + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbQueueManager := sbNamespace.NewQueueManager() + err = sbQueueManager.Delete(context.Background(), queueName) + require.NoErrorf(t, err, "cannot delete existing service bus queue - %s", err) +} diff --git a/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go b/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go new file mode 100644 index 00000000000..14f94bf29e1 --- /dev/null +++ b/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go @@ -0,0 +1,237 @@ +package azure_service_bus_topic_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + "time" + + servicebus "github.com/Azure/azure-service-bus-go" + "github.com/joho/godotenv" + "github.com/stretchr/testify/require" + + . "github.com/kedacore/keda/v2/tests" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "azure-service-bus-topic-test" +) + +var ( + connectionString = os.Getenv("AZURE_SERVICE_BUS_CONNECTION_STRING") + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + topicName = fmt.Sprintf("%s-topic", testName) + subscriptionName = fmt.Sprintf("%s-subscription", testName) +) + +type templateData struct { + TestNamespace string + SecretName string + Connection string + DeploymentName string + TriggerAuthName string + ScaledObjectName string + TopicName string + SubscriptionName string +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +type: Opaque +data: + connection: {{.Connection}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: nginx + image: nginx:1.16.1 +` + + triggerAuthTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: connection + name: {{.SecretName}} + key: connection +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} + labels: + deploymentName: {{.DeploymentName}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + cooldownPeriod: 10 + minReplicaCount: 0 + maxReplicaCount: 1 + triggers: + - type: azure-servicebus + metadata: + topicName: {{.TopicName}} + subscriptionName: {{.SubscriptionName}} + authenticationRef: + name: {{.TriggerAuthName}} +` +) + +func TestSetup(t *testing.T) { + require.NotEmpty(t, connectionString, "AZURE_SERVICE_BUS_CONNECTION_STRING env variable is required for service bus tests") + + // Connect to service bus namespace. + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbTopicManager := sbNamespace.NewTopicManager() + sbTopics, err := sbTopicManager.List(context.Background()) + require.NoErrorf(t, err, "cannot fetch topic list for service bus namespace - %s", err) + + // Delete service bus topic if already exists. + for _, topic := range sbTopics { + if topic.Name == topicName { + t.Log("Service Bus Topic already exists. Deleting.") + err := sbTopicManager.Delete(context.Background(), topicName) + require.NoErrorf(t, err, "cannot delete existing service bus topic - %s", err) + } + } + + // Create service bus topic. + _, err = sbTopicManager.Put(context.Background(), topicName) + require.NoErrorf(t, err, "cannot create service bus topic - %s", err) + + // Create subscription within topic + sbSubscriptionManager, err := sbNamespace.NewSubscriptionManager(topicName) + require.NoErrorf(t, err, "cannot create subscription manager for topic - %s", err) + + _, err = sbSubscriptionManager.Put(context.Background(), subscriptionName) + require.NoErrorf(t, err, "cannot create subscription for topic - %s", err) + + Kc = GetKubernetesClient(t) + CreateNamespace(t, Kc, testNamespace) + + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + data := templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + TopicName: topicName, + SubscriptionName: subscriptionName, + } + + // Create kubernetes resources + KubectlApplyMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") +} + +func TestScaleUp(t *testing.T) { + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbTopic, err := sbNamespace.NewTopic(topicName) + require.NoErrorf(t, err, "cannot create for topic - %s", err) + + for i := 0; i < 5; i++ { + _ = sbTopic.Send(context.Background(), servicebus.NewMessageFromString(fmt.Sprintf("Message - %d", i))) + } + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 1, 60, 1), + "replica count should be 1 after 1 minute") +} + +func TestScaleDown(t *testing.T) { + var messageHandlerFunc servicebus.HandlerFunc = func(ctx context.Context, msg *servicebus.Message) error { + return msg.Complete(ctx) + } + + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbTopic, err := sbNamespace.NewTopic(topicName) + require.NoErrorf(t, err, "cannot create client for topic - %s", err) + + sbSubscription, err := sbTopic.NewSubscription(subscriptionName) + require.NoErrorf(t, err, "cannot create client for subscription - %s", err) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _ = sbSubscription.Receive(ctx, messageHandlerFunc) + + require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after 1 minute") +} + +func TestCleanup(t *testing.T) { + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + data := templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + TopicName: topicName, + SubscriptionName: subscriptionName, + } + + // Delete kubernetes resources + KubectlDeleteMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) + + Kc = GetKubernetesClient(t) + DeleteNamespace(t, Kc, testNamespace) + + // Delete service bus topic. + sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) + require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + + sbTopicManager := sbNamespace.NewTopicManager() + err = sbTopicManager.Delete(context.Background(), topicName) + require.NoErrorf(t, err, "cannot delete existing service bus topic - %s", err) +} From 064b92ac3c9636945ab9ffddadba60a3a0e5c469 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 14:13:50 +0530 Subject: [PATCH 08/15] Add Queue (Trigger Auth), Key Vault Tests. Refactor old tests. Signed-off-by: Vighnesh Shenoy --- tests/helper.go | 12 +- tests/run-all.sh | 2 +- tests/scalers/azure-keyvault-queue.test.ts | 182 -------------- .../azure_keyvault_queue_test.go | 238 ++++++++++++++++++ .../azure_queue/azure_queue_test.go | 136 ++++------ .../azure_queue_trigger_auth_test.go | 211 ++++++++++++++++ .../azure_service_bus_queue_test.go | 118 +++++---- .../azure_service_bus_topic_test.go | 126 +++++----- 8 files changed, 626 insertions(+), 399 deletions(-) delete mode 100644 tests/scalers/azure-keyvault-queue.test.ts create mode 100644 tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go create mode 100644 tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go diff --git a/tests/helper.go b/tests/helper.go index ce29ae873e9..f5ec2bcfda6 100644 --- a/tests/helper.go +++ b/tests/helper.go @@ -30,7 +30,7 @@ const ( var _ = godotenv.Load() -// Env variables assertd for setup and cleanup. +// Env variables required for setup and cleanup. var ( AzureADTenantID = os.Getenv("AZURE_SP_TENANT") AzureRunWorkloadIdentityTests = os.Getenv("AZURE_RUN_WORKLOAD_IDENTITY_TESTS") @@ -184,3 +184,13 @@ func KubectlDeleteMultipleWithTemplate(t *testing.T, data interface{}, configs . KubectlDeleteWithTemplate(t, data, config) } } + +func CreateKubernetesResources(t *testing.T, kc *kubernetes.Clientset, nsName string, data interface{}, configs ...string) { + CreateNamespace(t, kc, nsName) + KubectlApplyMultipleWithTemplate(t, data, configs...) +} + +func DeleteKubernetesResources(t *testing.T, kc *kubernetes.Clientset, nsName string, data interface{}, configs ...string) { + DeleteNamespace(t, kc, nsName) + KubectlDeleteMultipleWithTemplate(t, data, configs...) +} diff --git a/tests/run-all.sh b/tests/run-all.sh index 8f3a0f0fab2..86a6ed3c02b 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -26,7 +26,7 @@ function run_tests { # TODO - Remove TypeScript regex after all tests have been migrated to Go. for test_case in $(find . -name "$E2E_REGEX_GO" -o -name "$E2E_REGEX_TS" | shuf) do - if [[ $test_case != *_test.go && $test_case != *.test.ts ]] # Skip helper files + if [[ $test_case != *_test.go ]] # Skip helper files then continue fi diff --git a/tests/scalers/azure-keyvault-queue.test.ts b/tests/scalers/azure-keyvault-queue.test.ts deleted file mode 100644 index 3374dbc29f2..00000000000 --- a/tests/scalers/azure-keyvault-queue.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -import * as async from 'async' -import * as azure from 'azure-storage' -import * as fs from 'fs' -import * as sh from 'shelljs' -import * as tmp from 'tmp' -import test from 'ava' -import {createNamespace, waitForDeploymentReplicaCount} from "./helpers"; - -const testNamespace = 'azure-keyvault-queue-test' -const queueName = 'queue-name-trigger' -const connectionString = process.env['AZURE_STORAGE_CONNECTION_STRING'] -const keyvaultURI = process.env['AZURE_KEYVAULT_URI'] -const azureADClientID = process.env['AZURE_SP_APP_ID'] -const azureADClientSecret = process.env['AZURE_SP_KEY'] -const azureADTenantID = process.env['AZURE_SP_TENANT'] - -test.before(async t => { - if (!connectionString) { - t.fail('AZURE_STORAGE_CONNECTION_STRING environment variable is required for keyvault tests') - } - - if (!keyvaultURI) { - t.fail('AZURE_KEYVAULT_URI environment variable is required for keyvault tests') - } - - if (!azureADClientID) { - t.fail('AZURE_SP_APP_ID environment variable is required for keyvault tests') - } - - if (!azureADClientSecret) { - t.fail('AZURE_SP_KEY environment variable is required for keyvault tests') - } - - if (!azureADTenantID) { - t.fail('AZURE_SP_TENANT environment variable is required for keyvault tests') - } - - const createQueueAsync = () => new Promise((resolve, _) => { - const queueSvc = azure.createQueueService(connectionString) - queueSvc.messageEncoder = new azure.QueueMessageEncoder.TextBase64QueueMessageEncoder() - queueSvc.createQueueIfNotExists(queueName, _ => { - resolve(undefined); - }) - }) - await createQueueAsync() - - sh.config.silent = true - const base64ConStr = Buffer.from(connectionString).toString('base64') - const base64ClientSecret = Buffer.from(azureADClientSecret).toString('base64') - - const tmpFile = tmp.fileSync() - fs.writeFileSync(tmpFile.name, deployYaml.replace(/{{CONNECTION_STRING_BASE64}}/g, base64ConStr) - .replace(/{{CLIENT_SECRET_BASE64}}/g, base64ClientSecret)) - - createNamespace(testNamespace) - t.is( - 0, - sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${testNamespace}`).code, - 'creating a deployment should work.' - ) - t.true(await waitForDeploymentReplicaCount(0, 'test-deployment', testNamespace, 60, 1000), 'replica count should be 0 after 1 minute') -}) - -test.serial( - 'Deployment should scale with messages on storage defined through trigger auth', - async t => { - const queueSvc = azure.createQueueService(connectionString) - queueSvc.messageEncoder = new azure.QueueMessageEncoder.TextBase64QueueMessageEncoder() - await async.mapLimit( - Array(1000).keys(), - 20, - (n, cb) => queueSvc.createMessage(queueName, `test ${n}`, cb) - ) - - // Scaling out when messages available - t.true(await waitForDeploymentReplicaCount(1, 'test-deployment', testNamespace, 60, 1000), 'replica count should be 1 after 1 minute') - - queueSvc.clearMessages(queueName, _ => {}) - - // Scaling in when no available messages - t.true(await waitForDeploymentReplicaCount(0, 'test-deployment', testNamespace, 300, 1000), 'replica count should be 0 after 5 minute') - } -) - -test.after.always.cb('clean up azure-queue deployment', t => { - const resources = [ - 'scaledobject.keda.sh/test-scaledobject', - 'triggerauthentications.keda.sh/azure-queue-auth', - 'secret/test-auth-secrets', - 'deployment.apps/test-deployment', - ] - - for (const resource of resources) { - sh.exec(`kubectl delete ${resource} --namespace ${testNamespace}`) - } - sh.exec(`kubectl delete namespace ${testNamespace}`) - - // delete test queue - const queueSvc = azure.createQueueService(connectionString) - queueSvc.deleteQueueIfExists(queueName, err => { - t.falsy(err, 'should delete test queue successfully') - t.end() - }) -}) - -const deployYaml = `apiVersion: apps/v1 -kind: Deployment -metadata: - name: test-deployment - labels: - app: test-deployment -spec: - replicas: 1 - selector: - matchLabels: - app: test-deployment - template: - metadata: - name: - namespace: - labels: - app: test-deployment - spec: - containers: - - name: test-deployment - image: ghcr.io/kedacore/tests-azure-queue - resources: - ports: - env: - - name: FUNCTIONS_WORKER_RUNTIME - value: node - - name: AzureWebJobsStorage - valueFrom: - secretKeyRef: - name: test-auth-secrets - key: connectionString ---- -apiVersion: v1 -kind: Secret -metadata: - name: test-auth-secrets - labels: -data: - connectionString: {{CONNECTION_STRING_BASE64}} - clientSecret: {{CLIENT_SECRET_BASE64}} ---- -apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: azure-keyvault-auth -spec: - azureKeyVault: - vaultUri: ${keyvaultURI} - credentials: - clientId: ${azureADClientID} - tenantId: ${azureADTenantID} - clientSecret: - valueFrom: - secretKeyRef: - name: test-auth-secrets - key: clientSecret - secrets: - - parameter: connection - name: E2E-Storage-ConnectionString ---- -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: test-scaledobject -spec: - scaleTargetRef: - name: test-deployment - pollingInterval: 5 - cooldownPeriod: 10 - minReplicaCount: 0 - maxReplicaCount: 1 - triggers: - - type: azure-queue - authenticationRef: - name: azure-keyvault-auth - metadata: - queueName: ${queueName}` diff --git a/tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go b/tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go new file mode 100644 index 00000000000..4d5219a04ca --- /dev/null +++ b/tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go @@ -0,0 +1,238 @@ +package azure_keyvault_queue_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/scalers/azure" + kedautil "github.com/kedacore/keda/v2/pkg/util" + . "github.com/kedacore/keda/v2/tests" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "azure-keyvault-queue-test" +) + +var ( + connectionString = os.Getenv("AZURE_STORAGE_CONNECTION_STRING") + keyvaultURI = os.Getenv("AZURE_KEYVAULT_URI") + azureADClientID = os.Getenv("AZURE_SP_APP_ID") + azureADSecret = os.Getenv("AZURE_SP_KEY") + azureADTenantID = os.Getenv("AZURE_SP_TENANT") + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + queueName = fmt.Sprintf("%s-queue", testName) +) + +type templateData struct { + TestNamespace string + SecretName string + Connection string + DeploymentName string + TriggerAuthName string + ScaledObjectName string + QueueName string + KeyVaultURI string + AzureADClientID string + AzureADSecret string + AzureADTenantID string +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + AzureWebJobsStorage: {{.Connection}} + clientSecret: {{.AzureADSecret}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: ghcr.io/kedacore/tests-azure-queue + resources: + env: + - name: FUNCTIONS_WORKER_RUNTIME + value: node + - name: AzureWebJobsStorage + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: AzureWebJobsStorage +` + + triggerAuthTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + azureKeyVault: + vaultUri: {{.KeyVaultURI}} + credentials: + clientId: {{.AzureADClientID}} + tenantId: {{.AzureADTenantID}} + clientSecret: + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: clientSecret + secrets: + - parameter: connection + name: E2E-Storage-ConnectionString +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: 1 + cooldownPeriod: 10 + triggers: + - type: azure-queue + metadata: + queueName: {{.QueueName}} + authenticationRef: + name: {{.TriggerAuthName}} +` +) + +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, connectionString, "AZURE_STORAGE_CONNECTION_STRING env variable is required for service bus tests") + require.NotEmpty(t, keyvaultURI, "AZURE_KEYVAULT_URI env variable is required for service bus tests") + require.NotEmpty(t, azureADClientID, "AZURE_SP_APP_ID env variable is required for service bus tests") + require.NotEmpty(t, azureADSecret, "AZURE_SP_KEY env variable is required for service bus tests") + require.NotEmpty(t, azureADTenantID, "AZURE_SP_TENANT env variable is required for service bus tests") + + queueURL, messageURL := createQueue(t) + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates...) + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") + + // test scaling + testScaleUp(t, kc, messageURL) + testScaleDown(t, kc, messageURL) + + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates...) + cleanupQueue(t, queueURL) +} + +func createQueue(t *testing.T) (azqueue.QueueURL, azqueue.MessagesURL) { + // Create Queue + httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) + credential, endpoint, err := azure.ParseAzureStorageQueueConnection( + context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") + assert.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + + p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) + serviceURL := azqueue.NewServiceURL(*endpoint, p) + queueURL := serviceURL.NewQueueURL(queueName) + + _, err = queueURL.Create(context.Background(), azqueue.Metadata{}) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) + + messageURL := queueURL.NewMessagesURL() + + return queueURL, messageURL +} + +func getTemplateData() (templateData, []string) { + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + base64ClientSecret := base64.StdEncoding.EncodeToString([]byte(azureADSecret)) + + return templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + QueueName: queueName, + KeyVaultURI: keyvaultURI, + AzureADClientID: azureADClientID, + AzureADSecret: base64ClientSecret, + AzureADTenantID: azureADTenantID, + }, []string{secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate} +} + +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale up ---") + for i := 0; i < 5; i++ { + msg := fmt.Sprintf("Message - %d", i) + _, err := messageURL.Enqueue(context.Background(), msg, 0*time.Second, time.Hour) + assert.NoErrorf(t, err, "cannot enqueue message - %s", err) + } + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 1, 60, 1), + "replica count should be 0 after a minute") +} + +func testScaleDown(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale down ---") + _, err := messageURL.Clear(context.Background()) + assert.NoErrorf(t, err, "cannot clear queue - %s", err) + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") +} + +func cleanupQueue(t *testing.T, queueURL azqueue.QueueURL) { + t.Log("--- cleaning up ---") + _, err := queueURL.Delete(context.Background()) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) +} diff --git a/tests/scalers_go/azure_queue/azure_queue_test.go b/tests/scalers_go/azure_queue/azure_queue_test.go index 64121c7f1a5..a21ffe05672 100644 --- a/tests/scalers_go/azure_queue/azure_queue_test.go +++ b/tests/scalers_go/azure_queue/azure_queue_test.go @@ -10,7 +10,9 @@ import ( "github.com/Azure/azure-storage-queue-go/azqueue" "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/kedacore/keda/v2/pkg/scalers/azure" @@ -107,130 +109,86 @@ spec: ` ) -func TestSetup(t *testing.T) { +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") require.NotEmpty(t, connectionString, "AZURE_STORAGE_CONNECTION_STRING env variable is required for service bus tests") - // Create Queue - httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) - credential, endpoint, err := azure.ParseAzureStorageQueueConnection( - context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") - require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) - - p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) - serviceURL := azqueue.NewServiceURL(*endpoint, p) - queueURL := serviceURL.NewQueueURL(queueName) - - _, err = queueURL.Create(context.Background(), azqueue.Metadata{}) - require.NoErrorf(t, err, "cannot create storage queue - %s", err) + queueURL, messageURL := createQueue(t) // Create kubernetes resources - Kc = GetKubernetesClient(t) - CreateNamespace(t, Kc, testNamespace) - - base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) - - data := templateData{ - TestNamespace: testNamespace, - SecretName: secretName, - Connection: base64ConnectionString, - DeploymentName: deploymentName, - ScaledObjectName: scaledObjectName, - QueueName: queueName, - } + kc := GetKubernetesClient(t) + data, templates := getTemplateData() - KubectlApplyMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, scaledObjectTemplate) + CreateKubernetesResources(t, kc, testNamespace, data, templates...) - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), "replica count should be 0 after a minute") -} -func TestScaleUp(t *testing.T) { - // Create Queue - httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) - credential, endpoint, err := azure.ParseAzureStorageQueueConnection( - context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") - require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + // test scaling + testScaleUp(t, kc, messageURL) + testScaleDown(t, kc, messageURL) - p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) - serviceURL := azqueue.NewServiceURL(*endpoint, p) - queueURL := serviceURL.NewQueueURL(queueName) - - messageURL := queueURL.NewMessagesURL() - - for i := 0; i < 5; i++ { - go func(t *testing.T, idx int) { - for j := 0; j < 200; j++ { - msg := fmt.Sprintf("Routine %d - Message - %d", idx, j) - _, err := messageURL.Enqueue(context.Background(), msg, 0*time.Second, time.Hour) - require.NoErrorf(t, err, "cannot enqueue message - %s", err) - } - }(t, i) - } - - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 1, 60, 1), - "replica count should be 0 after a minute") + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates...) + cleanupQueue(t, queueURL) } -func TestScaleDown(t *testing.T) { +func createQueue(t *testing.T) (azqueue.QueueURL, azqueue.MessagesURL) { // Create Queue httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) credential, endpoint, err := azure.ParseAzureStorageQueueConnection( context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") - require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + assert.NoErrorf(t, err, "cannot parse storage connection string - %s", err) p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) serviceURL := azqueue.NewServiceURL(*endpoint, p) queueURL := serviceURL.NewQueueURL(queueName) - messageURL := queueURL.NewMessagesURL() + _, err = queueURL.Create(context.Background(), azqueue.Metadata{}) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) - // Clear queue - // I am not sure if this is the best way to go about this. If someone can find a better way - // please raise a PR. - for { - props, err := queueURL.GetProperties(context.Background()) - require.NoErrorf(t, err, "cannot fetch queue properties - %s", err) - - if props.ApproximateMessagesCount() > 0 { - _, err := messageURL.Clear(context.Background()) - require.NoErrorf(t, err, "cannot clear storage queue - %s", err) - } else { - break - } - } + messageURL := queueURL.NewMessagesURL() - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), - "replica count should be 0 after a minute") + return queueURL, messageURL } -func TestCleanup(t *testing.T) { +func getTemplateData() (templateData, []string) { base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) - data := templateData{ + return templateData{ TestNamespace: testNamespace, SecretName: secretName, Connection: base64ConnectionString, DeploymentName: deploymentName, ScaledObjectName: scaledObjectName, QueueName: queueName, - } + }, []string{secretTemplate, deploymentTemplate, scaledObjectTemplate} +} - // Delete kubernetes resources - KubectlDeleteMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, scaledObjectTemplate) +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale up ---") + for i := 0; i < 5; i++ { + msg := fmt.Sprintf("Message - %d", i) + _, err := messageURL.Enqueue(context.Background(), msg, 0*time.Second, time.Hour) + assert.NoErrorf(t, err, "cannot enqueue message - %s", err) + } - Kc = GetKubernetesClient(t) - DeleteNamespace(t, Kc, testNamespace) + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 1, 60, 1), + "replica count should be 0 after a minute") +} - // Create Queue - httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) - credential, endpoint, err := azure.ParseAzureStorageQueueConnection( - context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") - require.NoErrorf(t, err, "cannot parse storage connection string - %s", err) +func testScaleDown(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale down ---") + _, err := messageURL.Clear(context.Background()) + assert.NoErrorf(t, err, "cannot clear queue - %s", err) - p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) - serviceURL := azqueue.NewServiceURL(*endpoint, p) - queueURL := serviceURL.NewQueueURL(queueName) + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") +} - _, err = queueURL.Delete(context.Background()) - require.NoErrorf(t, err, "cannot create storage queue - %s", err) +func cleanupQueue(t *testing.T, queueURL azqueue.QueueURL) { + t.Log("--- cleaning up ---") + _, err := queueURL.Delete(context.Background()) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) } diff --git a/tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go b/tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go new file mode 100644 index 00000000000..6cabdd9bcc9 --- /dev/null +++ b/tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go @@ -0,0 +1,211 @@ +package azure_queue_trigger_auth_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/scalers/azure" + kedautil "github.com/kedacore/keda/v2/pkg/util" + . "github.com/kedacore/keda/v2/tests" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "azure-queue-trigger-auth-test" +) + +var ( + connectionString = os.Getenv("AZURE_STORAGE_CONNECTION_STRING") + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + queueName = fmt.Sprintf("%s-queue", testName) +) + +type templateData struct { + TestNamespace string + SecretName string + Connection string + DeploymentName string + TriggerAuthName string + ScaledObjectName string + QueueName string +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + AzureWebJobsStorage: {{.Connection}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: ghcr.io/kedacore/tests-azure-queue + resources: + env: + - name: FUNCTIONS_WORKER_RUNTIME + value: node + - name: AzureWebJobsStorage + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: AzureWebJobsStorage +` + + triggerAuthTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: connection + name: {{.SecretName}} + key: AzureWebJobsStorage +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: 1 + cooldownPeriod: 10 + triggers: + - type: azure-queue + metadata: + queueName: {{.QueueName}} + authenticationRef: + name: {{.TriggerAuthName}} +` +) + +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, connectionString, "AZURE_STORAGE_CONNECTION_STRING env variable is required for service bus tests") + + queueURL, messageURL := createQueue(t) + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates...) + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") + + // test scaling + testScaleUp(t, kc, messageURL) + testScaleDown(t, kc, messageURL) + + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates...) + cleanupQueue(t, queueURL) +} + +func createQueue(t *testing.T) (azqueue.QueueURL, azqueue.MessagesURL) { + // Create Queue + httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) + credential, endpoint, err := azure.ParseAzureStorageQueueConnection( + context.Background(), httpClient, kedav1alpha1.PodIdentityProviderNone, connectionString, "", "") + assert.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + + p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) + serviceURL := azqueue.NewServiceURL(*endpoint, p) + queueURL := serviceURL.NewQueueURL(queueName) + + _, err = queueURL.Create(context.Background(), azqueue.Metadata{}) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) + + messageURL := queueURL.NewMessagesURL() + + return queueURL, messageURL +} + +func getTemplateData() (templateData, []string) { + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + return templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + QueueName: queueName, + }, []string{secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate} +} + +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale up ---") + for i := 0; i < 5; i++ { + msg := fmt.Sprintf("Message - %d", i) + _, err := messageURL.Enqueue(context.Background(), msg, 0*time.Second, time.Hour) + assert.NoErrorf(t, err, "cannot enqueue message - %s", err) + } + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 1, 60, 1), + "replica count should be 0 after a minute") +} + +func testScaleDown(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale down ---") + _, err := messageURL.Clear(context.Background()) + assert.NoErrorf(t, err, "cannot clear queue - %s", err) + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") +} + +func cleanupQueue(t *testing.T, queueURL azqueue.QueueURL) { + t.Log("--- cleaning up ---") + _, err := queueURL.Delete(context.Background()) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) +} diff --git a/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go b/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go index 25129f9d955..3c402fcd510 100644 --- a/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go +++ b/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go @@ -10,7 +10,9 @@ import ( servicebus "github.com/Azure/azure-service-bus-go" "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" . "github.com/kedacore/keda/v2/tests" ) @@ -112,36 +114,67 @@ spec: ` ) -func TestSetup(t *testing.T) { +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") require.NotEmpty(t, connectionString, "AZURE_SERVICE_BUS_CONNECTION_STRING env variable is required for service bus tests") + sbQueueManager, sbQueue := setupServiceBusQueue(t) + + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates...) + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") + + // test scaling + testScaleUp(t, kc, sbQueue) + testScaleDown(t, kc, sbQueue) + + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates...) + cleanupServiceBusQueue(t, sbQueueManager) +} + +func setupServiceBusQueue(t *testing.T) (*servicebus.QueueManager, *servicebus.Queue) { // Connect to service bus namespace. sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + assert.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) sbQueueManager := sbNamespace.NewQueueManager() + + createQueue(t, sbQueueManager) + + sbQueue, err := sbNamespace.NewQueue(queueName) + assert.NoErrorf(t, err, "cannot create client for queue - %s", err) + + return sbQueueManager, sbQueue +} + +func createQueue(t *testing.T, sbQueueManager *servicebus.QueueManager) { + // delete queue if already exists sbQueues, err := sbQueueManager.List(context.Background()) - require.NoErrorf(t, err, "cannot fetch queue list for service bus namespace - %s", err) + assert.NoErrorf(t, err, "cannot fetch queue list for service bus namespace - %s", err) - // Delete service bus queue if already exists. for _, queue := range sbQueues { if queue.Name == queueName { t.Log("Service Bus Queue already exists. Deleting.") err := sbQueueManager.Delete(context.Background(), queueName) - require.NoErrorf(t, err, "cannot delete existing service bus queue - %s", err) + assert.NoErrorf(t, err, "cannot delete existing service bus queue - %s", err) } } - // Create service bus queue. + // create queue _, err = sbQueueManager.Put(context.Background(), queueName) - require.NoErrorf(t, err, "cannot create service bus queue - %s", err) - - Kc = GetKubernetesClient(t) - CreateNamespace(t, Kc, testNamespace) + assert.NoErrorf(t, err, "cannot create service bus queue - %s", err) +} +func getTemplateData() (templateData, []string) { base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) - data := templateData{ + return templateData{ TestNamespace: testNamespace, SecretName: secretName, Connection: base64ConnectionString, @@ -149,74 +182,37 @@ func TestSetup(t *testing.T) { TriggerAuthName: triggerAuthName, ScaledObjectName: scaledObjectName, QueueName: queueName, - } - - // Create kubernetes resources - KubectlApplyMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) - - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), - "replica count should be 0 after a minute") + }, []string{secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate} } -func TestScaleUp(t *testing.T) { - sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) - - sbQueue, err := sbNamespace.NewQueue(queueName) - require.NoErrorf(t, err, "cannot create client for queue - %s", err) - +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, sbQueue *servicebus.Queue) { + t.Log("--- testing scale up ---") for i := 0; i < 5; i++ { - _ = sbQueue.Send(context.Background(), servicebus.NewMessageFromString(fmt.Sprintf("Message - %d", i))) + msg := fmt.Sprintf("Message - %d", i) + _ = sbQueue.Send(context.Background(), servicebus.NewMessageFromString(msg)) } - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 1, 60, 1), + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 1, 60, 1), "replica count should be 1 after 1 minute") } -func TestScaleDown(t *testing.T) { +func testScaleDown(t *testing.T, kc *kubernetes.Clientset, sbQueue *servicebus.Queue) { + t.Log("--- testing scale down ---") var messageHandlerFunc servicebus.HandlerFunc = func(ctx context.Context, msg *servicebus.Message) error { return msg.Complete(ctx) } - sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) - - sbQueue, err := sbNamespace.NewQueue(queueName) - require.NoErrorf(t, err, "cannot create client for queue - %s", err) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = sbQueue.Receive(ctx, messageHandlerFunc) - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), "replica count should be 0 after 1 minute") } -func TestCleanup(t *testing.T) { - base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) - - data := templateData{ - TestNamespace: testNamespace, - SecretName: secretName, - Connection: base64ConnectionString, - DeploymentName: deploymentName, - TriggerAuthName: triggerAuthName, - ScaledObjectName: scaledObjectName, - QueueName: queueName, - } - - // Delete kubernetes resources - KubectlDeleteMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) - - Kc = GetKubernetesClient(t) - DeleteNamespace(t, Kc, testNamespace) - - // Delete service bus queue. - sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) - - sbQueueManager := sbNamespace.NewQueueManager() - err = sbQueueManager.Delete(context.Background(), queueName) - require.NoErrorf(t, err, "cannot delete existing service bus queue - %s", err) +func cleanupServiceBusQueue(t *testing.T, sbQueueManager *servicebus.QueueManager) { + t.Log("--- cleaning up ---") + err := sbQueueManager.Delete(context.Background(), queueName) + assert.NoErrorf(t, err, "cannot delete service bus queue - %s", err) } diff --git a/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go b/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go index 14f94bf29e1..5396f5ce33a 100644 --- a/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go +++ b/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go @@ -10,7 +10,9 @@ import ( servicebus "github.com/Azure/azure-service-bus-go" "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" . "github.com/kedacore/keda/v2/tests" ) @@ -115,43 +117,78 @@ spec: ` ) -func TestSetup(t *testing.T) { +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") require.NotEmpty(t, connectionString, "AZURE_SERVICE_BUS_CONNECTION_STRING env variable is required for service bus tests") + sbTopicManager, sbTopic, sbSubscription := setupServiceBusTopicAndSubscription(t) + + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates...) + + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") + + // test scaling + testScaleUp(t, kc, sbTopic) + testScaleDown(t, kc, sbSubscription) + + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates...) + cleanupServiceBusTopic(t, sbTopicManager) +} + +func setupServiceBusTopicAndSubscription(t *testing.T) (*servicebus.TopicManager, *servicebus.Topic, *servicebus.Subscription) { // Connect to service bus namespace. sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) + assert.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) sbTopicManager := sbNamespace.NewTopicManager() + + createTopicAndSubscription(t, sbNamespace, sbTopicManager) + + sbTopic, err := sbNamespace.NewTopic(topicName) + assert.NoErrorf(t, err, "cannot create client for topic - %s", err) + + sbSubscription, err := sbTopic.NewSubscription(subscriptionName) + assert.NoErrorf(t, err, "cannot create client for subscription - %s", err) + + return sbTopicManager, sbTopic, sbSubscription +} + +func createTopicAndSubscription(t *testing.T, sbNamespace *servicebus.Namespace, sbTopicManager *servicebus.TopicManager) { + // Delete service bus topic if already exists. sbTopics, err := sbTopicManager.List(context.Background()) - require.NoErrorf(t, err, "cannot fetch topic list for service bus namespace - %s", err) + assert.NoErrorf(t, err, "cannot fetch topic list for service bus namespace - %s", err) // Delete service bus topic if already exists. for _, topic := range sbTopics { if topic.Name == topicName { t.Log("Service Bus Topic already exists. Deleting.") err := sbTopicManager.Delete(context.Background(), topicName) - require.NoErrorf(t, err, "cannot delete existing service bus topic - %s", err) + assert.NoErrorf(t, err, "cannot delete existing service bus topic - %s", err) } } // Create service bus topic. _, err = sbTopicManager.Put(context.Background(), topicName) - require.NoErrorf(t, err, "cannot create service bus topic - %s", err) + assert.NoErrorf(t, err, "cannot create service bus topic - %s", err) // Create subscription within topic sbSubscriptionManager, err := sbNamespace.NewSubscriptionManager(topicName) - require.NoErrorf(t, err, "cannot create subscription manager for topic - %s", err) + assert.NoErrorf(t, err, "cannot create subscription manager for topic - %s", err) _, err = sbSubscriptionManager.Put(context.Background(), subscriptionName) - require.NoErrorf(t, err, "cannot create subscription for topic - %s", err) - - Kc = GetKubernetesClient(t) - CreateNamespace(t, Kc, testNamespace) + assert.NoErrorf(t, err, "cannot create subscription for topic - %s", err) +} +func getTemplateData() (templateData, []string) { base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) - data := templateData{ + return templateData{ TestNamespace: testNamespace, SecretName: secretName, Connection: base64ConnectionString, @@ -160,78 +197,37 @@ func TestSetup(t *testing.T) { ScaledObjectName: scaledObjectName, TopicName: topicName, SubscriptionName: subscriptionName, - } - - // Create kubernetes resources - KubectlApplyMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) - - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), - "replica count should be 0 after a minute") + }, []string{secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate} } -func TestScaleUp(t *testing.T) { - sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) - - sbTopic, err := sbNamespace.NewTopic(topicName) - require.NoErrorf(t, err, "cannot create for topic - %s", err) - +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, sbTopic *servicebus.Topic) { + t.Log("--- testing scale up ---") for i := 0; i < 5; i++ { - _ = sbTopic.Send(context.Background(), servicebus.NewMessageFromString(fmt.Sprintf("Message - %d", i))) + msg := fmt.Sprintf("Message - %d", i) + _ = sbTopic.Send(context.Background(), servicebus.NewMessageFromString(msg)) } - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 1, 60, 1), + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 1, 60, 1), "replica count should be 1 after 1 minute") } -func TestScaleDown(t *testing.T) { +func testScaleDown(t *testing.T, kc *kubernetes.Clientset, sbSubscription *servicebus.Subscription) { + t.Log("--- testing scale down ---") var messageHandlerFunc servicebus.HandlerFunc = func(ctx context.Context, msg *servicebus.Message) error { return msg.Complete(ctx) } - sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) - - sbTopic, err := sbNamespace.NewTopic(topicName) - require.NoErrorf(t, err, "cannot create client for topic - %s", err) - - sbSubscription, err := sbTopic.NewSubscription(subscriptionName) - require.NoErrorf(t, err, "cannot create client for subscription - %s", err) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = sbSubscription.Receive(ctx, messageHandlerFunc) - require.True(t, WaitForDeploymentReplicaCount(t, Kc, deploymentName, testNamespace, 0, 60, 1), + assert.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 0, 60, 1), "replica count should be 0 after 1 minute") } -func TestCleanup(t *testing.T) { - base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) - - data := templateData{ - TestNamespace: testNamespace, - SecretName: secretName, - Connection: base64ConnectionString, - DeploymentName: deploymentName, - TriggerAuthName: triggerAuthName, - ScaledObjectName: scaledObjectName, - TopicName: topicName, - SubscriptionName: subscriptionName, - } - - // Delete kubernetes resources - KubectlDeleteMultipleWithTemplate(t, data, secretTemplate, deploymentTemplate, triggerAuthTemplate, scaledObjectTemplate) - - Kc = GetKubernetesClient(t) - DeleteNamespace(t, Kc, testNamespace) - - // Delete service bus topic. - sbNamespace, err := servicebus.NewNamespace(servicebus.NamespaceWithConnectionString(connectionString)) - require.NoErrorf(t, err, "cannot connect to service bus namespace - %s", err) - - sbTopicManager := sbNamespace.NewTopicManager() - err = sbTopicManager.Delete(context.Background(), topicName) - require.NoErrorf(t, err, "cannot delete existing service bus topic - %s", err) +func cleanupServiceBusTopic(t *testing.T, sbTopicManager *servicebus.TopicManager) { + t.Log("--- cleaning up ---") + err := sbTopicManager.Delete(context.Background(), topicName) + assert.NoErrorf(t, err, "cannot delete service bus topic - %s", err) } From f6b4d2a87e7b5549acbacc0cb189aa66f98fc6b9 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 15:07:03 +0530 Subject: [PATCH 09/15] Update README. Signed-off-by: Vighnesh Shenoy --- tests/OLD-README.md | 130 ++++++++++++++++++++++++++ tests/README.md | 219 +++++++++++++++++++++++++++----------------- 2 files changed, 266 insertions(+), 83 deletions(-) create mode 100644 tests/OLD-README.md diff --git a/tests/OLD-README.md b/tests/OLD-README.md new file mode 100644 index 00000000000..e770dca8622 --- /dev/null +++ b/tests/OLD-README.md @@ -0,0 +1,130 @@ +> âš âš  **Important:** âš âš  +> - E2E tests are currently in the process of being migrated to Go. +> - Refer to [this](https://github.com/kedacore/keda/issues/2737) issue for more information. +> - We are no longer accepting new tests written in TypeScript. +> - Please refer [README](README.md) for instructions on writing the E2E tests in Go. + +## Prerequisites + +- [node](https://nodejs.org/en/) +- `kubectl` logged into a Kubernetes cluster. +- Each scaler test might define additional requirements. For example, `azure-queue.test.ts` requires an env var `AZURE_STORAGE_CONNECTION_STRING` + +## Running tests: + +```bash +npm install +npm test --verbose +``` + +### Run one test file: + +``` +npx ava scalers/prometheus.test.ts +``` + +## E2E test setup + +The test script will run 3 phases: +- **Setup**: this is done in [`setup.test.ts`](setup.test.ts). If you're adding any tests for KEDA install/setup process add it to this file.`setup.test.ts` deploys KEDA to `keda` namespace in the cluster, and updates the image to `kedacore/keda:main`. + + After `setup.test.ts` is done, we expect to have a cluster with KEDA setup in namespace `keda`. This is done through a `pretest` hook in npm. See [`"scripts"` in package.json](package.json#L14). + +- **Tests**: Currently there are only scaler tests in `tests/scalers`. All files run in parallel, but tests within the file can run either in parallel or in series. More about tests below. + +- **Global clean up**: this is done in [`cleanup.test.ts`](cleanup.test.ts). This step cleans resources created in `setup.test.ts`. + + +## Adding tests: + +* Tests are written in TypeScript using [ava](https://github.com/avajs/ava) framework. See [ava docs here](https://github.com/avajs/ava/tree/main/docs) +* Each scaler tests should be in a file. **e.g**: `azure-queue.tests.ts`, `kafka.tests.ts`, etc +* All files in `scalers/**.ts` are run in parallel by default. Make sure your tests don't affect the global state of the cluster in a way that can break other tests. +* Each test file is expected to do it's own setup and clean up for its resources. + +```ts +// import test from ava framework +import test from 'ava'; + +test.before(t => { + // this runs once before all tests. + // do setup here. e.g: + // - Create a namespace for your tests using the function createNamespace(namespace: string) available in helpers file + // - Create deployment (using kubectl or kubernetes node-client) + // - Setup event source (deploy redis, or configure azure storage, etc) + // - etc +}); + + +// `test 1` and `test 2` will run in parallel. +test('test 1', t => { }); +test('test 2', t => { }); + +// `test 3` will run first, then `test 4`. +// Tests will run in the order they are defined in. +// All serial tests will run first before parallel tests above +test.serial('test 3', t => { }); +test.serial('test 4', t => { }); + +// Tests are expected to finish synchronously, or using async/await +// if you need to use callbacks, then add `.cb` and call `t.end()` when done. +test('test 6', t => { }); +test('test 7', async t => { }); +test.cb('test 8', t => { t.end() }); + +test.after.always.cb('clean up always after all tests', t => { + // Clean up after your test here. without `always` this will only run if all tests are successful. + t.end(); +}); +``` +>âš âš  **Important:** âš âš  Even thought the cleaning of the resources is expected inside each e2e test file, all test namespaces are cleaned up to ensure not having dangling resources after global e2e execution finishes. For not breaking this behavior, it's mandatory the usage of the function `createNamespace(namespace: string)` instead of creating them manually. + +* **Example test:** for example if I want to add a test for redis + +```ts +import * as sh from 'shelljs'; +import test from 'ava'; +import { createNamespace } from './helpers'; + +// you can include template in the file or in another file. +const deployYaml = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment + labels: + app: test-deployment +spec: + replicas: 0 + ... + ... + ` + +test.before('install redis and create deployment' t => { + if (!sh.which('helm')) { + t.fail('redis tests require helm'); + } + + sh.exec('helm install redis ......'); // install redis to the cluster + createNamespace("redis-test-deployment") + sh.exec('kubectl apply -f ....'); // create your deployment +}); + +test.serial('deployment should scale when adding items in redis list', t => { + // use node redis client to add stuff to redis. + // maybe sleep or poll the replica count + const replicaCount = sh.exec(`kubectl get deployment.apps/test-deployment .. -o jsonpath="{.spec.replicas}"`).stdout; + t.is('10', replicaCount, 'expecting replica count to be 10'); +}); + +test.after.always('remove redis and my deployment', t => { + sh.exec('kubectl delete ....'); +}); +``` + +* You can see [`azure-queue.test.ts`](scalers/azure-queue.test.ts) for a full example. +* Ava has more options for asserting and writing tests. The docs are very good. https://github.com/avajs/ava/blob/main/docs/01-writing-tests.md +* **debugging**: when debugging, you can force only 1 test to run by adding `only` to the test definition. + +```ts +test.serial.only('this will be the only test to run', t => { }); +``` diff --git a/tests/README.md b/tests/README.md index 337d3572d24..216a763609e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,87 +1,101 @@ ## Prerequisites -- [node](https://nodejs.org/en/) +- [go](https://go.dev/) - `kubectl` logged into a Kubernetes cluster. -- Each scaler test might define additional requirements. For example, `azure-queue.test.ts` requires an env var `AZURE_STORAGE_CONNECTION_STRING` +- Each scaler test might define additional requirements. For example, `azure_queue_test.go` requires an env var `AZURE_STORAGE_CONNECTION_STRING` ## Running tests: +### All tests + ```bash -npm install -npm test --verbose +go test -v setup_test.go # Only needs to be run once. +go test -v ./scalers_go/... +go test -v cleanup_test.go # Skip if you want to keep testing. ``` -### Run one test file: +### Specific test -``` -npx ava scalers/prometheus.test.ts +```bash +go test -v ./scalers_go/azure_queue/azure_queue_test.go # Assumes that setup has been run before ``` -## E2E test setup +Refer to [this](https://pkg.go.dev/testing) for more information about testing in `Go`. -The test script will run 3 phases: -- **Setup**: this is done in [`setup.test.ts`](setup.test.ts). If you're adding any tests for KEDA install/setup process add it to this file.`setup.test.ts` deploys KEDA to `keda` namespace in the cluster, and updates the image to `kedacore/keda:main`. +## E2E Test Setup - After `setup.test.ts` is done, we expect to have a cluster with KEDA setup in namespace `keda`. This is done through a `pretest` hook in npm. See [`"scripts"` in package.json](package.json#L14). +The test script will run in 3 phases: -- **Tests**: Currently there are only scaler tests in `tests/scalers`. All files run in parallel, but tests within the file can run either in parallel or in series. More about tests below. +- **Setup:** This is done in [`setup_test.go`](setup_test.go). If you're adding any tests to the KEDA install / setup process, you need to add it to this file. `setup_test.go` deploys KEDA to the `keda` namespace, updates the image to +`kedacore/keda:main`. -- **Global clean up**: this is done in [`cleanup.test.ts`](cleanup.test.ts). This step cleans resources created in `setup.test.ts`. + After `setup_test.go` is done, we expect to have KEDA setup in the `keda` namespace. +- **Tests:** Currently there are only scaler tests in `tests/scalers_go/`. Each test is kept in its own package. This is to prevent conflicting variable declarations for commoly used variables (**ex -** `testNamespace`). Individual scaler tests are run +in parallel, but tests within a file can be run in parallel or in series. More about tests below. -## Adding tests: +- **Global cleanup:** This is done in [`cleanup_test.go`](cleanup_test.go). It cleans up all the resources created in `setup_test.go`. -* Tests are written in TypeScript using [ava](https://github.com/avajs/ava) framework. See [ava docs here](https://github.com/avajs/ava/tree/main/docs) -* Each scaler tests should be in a file. **e.g**: `azure-queue.tests.ts`, `kafka.tests.ts`, etc -* All files in `scalers/**.ts` are run in parallel by default. Make sure your tests don't affect the global state of the cluster in a way that can break other tests. -* Each test file is expected to do it's own setup and clean up for its resources. +## Adding tests -```ts -// import test from ava framework -import test from 'ava'; +- Tests are written using `Go`'s default [`testing`](https://pkg.go.dev/testing) framework, and [`testify`](https://pkg.go.dev/github.com/stretchr/testify). +- Each scaler should be in its own package, **ex -** `scalers_go/azure_queue/azure_queue_test.go`, or `scalers_go/kafka/kafka_test.go`, etc +- Each test file is expected to do its own setup and clean for resources. -test.before(t => { - // this runs once before all tests. - // do setup here. e.g: - // - Create a namespace for your tests using the function createNamespace(namespace: string) available in helpers file - // - Create deployment (using kubectl or kubernetes node-client) - // - Setup event source (deploy redis, or configure azure storage, etc) - // - etc -}); +#### âš âš  Important: âš âš  +> +> - Even though the cleaning of resources is expected inside each e2e test file, all test namespaces +> (namespaces with label type=e2e) are cleaned up to ensure not having dangling resources after global e2e +> execution finishes. To not break this behaviour, it's mandatory to use the `CreateNamespace(t *testing.T, kc *kubernetes.Clientset, nsName string)` function from [`helper.go`](helper.go), instead of creating them manually. +#### âš âš  Important: âš âš  +> - `Go` code can panic when performing forbidden operations such as accessing a nil pointer, or from code that +> manually calls `panic()`. A function that `panics` passes the `panic` up the stack until program execution stops +> or it is recovered using `recover()` (somewhat similar to `try-catch` in other languages). +> - If a test panics, and is not recovered, `Go` will stop running the file, and no further tests will be run. This can +> cause issues with clean up of resources. +> - Ensure that you are not executing code that can lead to `panics`. If you think that there's a chance the test might +> panic, call `recover()`, and cleanup the created resources. +> - Read this [article](https://go.dev/blog/defer-panic-and-recover) for understanding more about `panic` and `recover` in `Go`. -// `test 1` and `test 2` will run in parallel. -test('test 1', t => { }); -test('test 2', t => { }); +#### **Example Test:** Let's say you want to add a test for `Redis`. -// `test 3` will run first, then `test 4`. -// Tests will run in the order they are defined in. -// All serial tests will run first before parallel tests above -test.serial('test 3', t => { }); -test.serial('test 4', t => { }); +```go +import ( + "context" + "fmt" + "os" + "testing" -// Tests are expected to finish synchronously, or using async/await -// if you need to use callbacks, then add `.cb` and call `t.end()` when done. -test('test 6', t => { }); -test('test 7', async t => { }); -test.cb('test 8', t => { t.end() }); + "github.com/joho/godotenv" + "github.com/stretchr/testify/require" + // Other required imports + ... + ... -test.after.always.cb('clean up always after all tests', t => { - // Clean up after your test here. without `always` this will only run if all tests are successful. - t.end(); -}); -``` ->âš âš  **Important:** âš âš  Even thought the cleaning of the resources is expected inside each e2e test file, all test namespaces are cleaned up to ensure not having dangling resources after global e2e execution finishes. For not breaking this behavior, it's mandatory the usage of the function `createNamespace(namespace: string)` instead of creating them manually. + . "github.com/kedacore/keda/v2/tests" // For helper methods +) + +var _ = godotenv.Load("../../.env") // For loading env variables from .env -* **Example test:** for example if I want to add a test for redis +const ( + testName = "redis-test" + // Other constants required for your test + ... + ... +) -```ts -import * as sh from 'shelljs'; -import test from 'ava'; -import { createNamespace } from './helpers'; +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + // Other variables required for your test + ... + ... +) -// you can include template in the file or in another file. -const deployYaml = `apiVersion: apps/v1 +// YAML templates for your Kubernetes resources +const ( + deploymentTemplate = ` +apiVersion: apps/v1 kind: Deployment metadata: name: test-deployment @@ -91,34 +105,73 @@ spec: replicas: 0 ... ... - ` - -test.before('install redis and create deployment' t => { - if (!sh.which('helm')) { - t.fail('redis tests require helm'); - } - - sh.exec('helm install redis ......'); // install redis to the cluster - createNamespace("redis-test-deployment") - sh.exec('kubectl apply -f ....'); // create your deployment -}); - -test.serial('deployment should scale when adding items in redis list', t => { - // use node redis client to add stuff to redis. - // maybe sleep or poll the replica count - const replicaCount = sh.exec(`kubectl get deployment.apps/test-deployment .. -o jsonpath="{.spec.replicas}"`).stdout; - t.is('10', replicaCount, 'expecting replica count to be 10'); -}); - -test.after.always('remove redis and my deployment', t => { - sh.exec('kubectl delete ....'); -}); +` +... +... +) + +type templateData struct { + // Fields used in your Kubernetes YAML templates + ... + ... +} + +func TestScaler(t *testing.T) { + setupTest(t) + + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates...) + + testScaleUp(t) + + // Ensure that this gets run. Using defer is necessary + DeleteKubernetesResources(t, kc, testNamespace, data, templates...) + cleanupTest(t) +} + +func setupTest(t *testing.T) { + t.Log("--- setting up ---") + _, err := ParseCommand("which helm").Output() + assert.NoErrorf(t, err, "redis test requires helm - %s", err) + + _, err := ParseCommand("helm install redis .....").Output() + assert.NoErrorf(t, err, "error while installing redis - %s", err) +} + +func getTemplateData() (templateData, []string) { + return templateData{ + // Populate fields required in YAML templates + ... + ... + }, []string{deploymentTemplate, scaledObjectTemplate} +} + +func testScaleUp(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale up ---") + // Use Go Redis Library to add stuff to redis to trigger scale up. + ... + ... + // Sleep / poll for replica count using helper method. + require.True(t, WaitForDeploymentReplicaCount(t, kc, deploymentName, testNamespace, 10, 60, 1), + "replica count should be 10 after 1 minute") +} + +func cleanupTest(t *testing.T) { + t.Log("--- cleaning up ---") + // Cleanup external resources (such as Blob Storage Container, RabbitMQ queue, Redis in this case) + ... + ... +} ``` -* You can see [`azure-queue.test.ts`](scalers/azure-queue.test.ts) for a full example. -* Ava has more options for asserting and writing tests. The docs are very good. https://github.com/avajs/ava/blob/main/docs/01-writing-tests.md -* **debugging**: when debugging, you can force only 1 test to run by adding `only` to the test definition. +#### Notes -```ts -test.serial.only('this will be the only test to run', t => { }); -``` +- You can see [`azure_queue_test.go`](scalers_go/azure_queue/azure_queue_test.go) for a full example. +- Refer [`helper.go`](helper.go) for various helper methods available to use in your tests. +- Prefer using helper methods or `k8s` libraries in `Go` over manually executing `shell` commands. Only if the task +you're trying to achieve is too complicated or tedious using above, use `ParseCommand` or `ExecuteCommand` from `helper.go` +for executing shell commands. +- Ensure, ensure, ensure that you're cleaning up resources. +- You can use `VS Code` for easily debugging your tests. From def558b09e60592634777604bd4e3aa065803409 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 15:35:06 +0530 Subject: [PATCH 10/15] Add missing build tags. Signed-off-by: Vighnesh Shenoy --- report.xml | 307 ++++++++++++++++++ tests/README.md | 3 + .../azure_keyvault_queue_test.go | 3 + .../azure_queue/azure_queue_test.go | 3 + .../azure_queue_trigger_auth_test.go | 3 + .../azure_service_bus_queue_test.go | 3 + .../azure_service_bus_topic_test.go | 3 + 7 files changed, 325 insertions(+) create mode 100644 report.xml diff --git a/report.xml b/report.xml new file mode 100644 index 00000000000..b44ac0681d3 --- /dev/null +++ b/report.xml @@ -0,0 +1,307 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/README.md b/tests/README.md index 216a763609e..d8c3fbd5e4a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -61,6 +61,8 @@ in parallel, but tests within a file can be run in parallel or in series. More a #### **Example Test:** Let's say you want to add a test for `Redis`. ```go +// +build e2e +// ^ This is necessary to ensure the tests don't get run in the GitHub workflow. import ( "context" "fmt" @@ -169,6 +171,7 @@ func cleanupTest(t *testing.T) { #### Notes - You can see [`azure_queue_test.go`](scalers_go/azure_queue/azure_queue_test.go) for a full example. +- All tests must have the `// +build e2e` build tag. - Refer [`helper.go`](helper.go) for various helper methods available to use in your tests. - Prefer using helper methods or `k8s` libraries in `Go` over manually executing `shell` commands. Only if the task you're trying to achieve is too complicated or tedious using above, use `ParseCommand` or `ExecuteCommand` from `helper.go` diff --git a/tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go b/tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go index 4d5219a04ca..8a6eeddf79e 100644 --- a/tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go +++ b/tests/scalers_go/azure_keyvault_queue/azure_keyvault_queue_test.go @@ -1,3 +1,6 @@ +//go:build e2e +// +build e2e + package azure_keyvault_queue_test import ( diff --git a/tests/scalers_go/azure_queue/azure_queue_test.go b/tests/scalers_go/azure_queue/azure_queue_test.go index a21ffe05672..1ec8b4a9620 100644 --- a/tests/scalers_go/azure_queue/azure_queue_test.go +++ b/tests/scalers_go/azure_queue/azure_queue_test.go @@ -1,3 +1,6 @@ +//go:build e2e +// +build e2e + package azure_queue_test import ( diff --git a/tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go b/tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go index 6cabdd9bcc9..f9180ce0c0c 100644 --- a/tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go +++ b/tests/scalers_go/azure_queue_trigger_auth/azure_queue_trigger_auth_test.go @@ -1,3 +1,6 @@ +//go:build e2e +// +build e2e + package azure_queue_trigger_auth_test import ( diff --git a/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go b/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go index 3c402fcd510..090fae81f4b 100644 --- a/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go +++ b/tests/scalers_go/azure_service_bus_queue/azure_service_bus_queue_test.go @@ -1,3 +1,6 @@ +//go:build e2e +// +build e2e + package azure_service_bus_queue_test import ( diff --git a/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go b/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go index 5396f5ce33a..26d1138c1c5 100644 --- a/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go +++ b/tests/scalers_go/azure_service_bus_topic/azure_service_bus_topic_test.go @@ -1,3 +1,6 @@ +//go:build e2e +// +build e2e + package azure_service_bus_topic_test import ( From d8b4786ff557b6856c11e89a41c760a7e5873d56 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 15:41:28 +0530 Subject: [PATCH 11/15] Fix run-all.sh. Signed-off-by: Vighnesh Shenoy --- tests/run-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run-all.sh b/tests/run-all.sh index 86a6ed3c02b..49e6f861510 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -26,7 +26,7 @@ function run_tests { # TODO - Remove TypeScript regex after all tests have been migrated to Go. for test_case in $(find . -name "$E2E_REGEX_GO" -o -name "$E2E_REGEX_TS" | shuf) do - if [[ $test_case != *_test.go ]] # Skip helper files + if [[ $test_case != *_test.go && $test_case != *.test.ts ]] # Skip helper files then continue fi From 0b6dcd9a5b1d6c31c4d62559b2146cc5741e4cb2 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 17:06:57 +0530 Subject: [PATCH 12/15] Fix unbound variable error. Signed-off-by: Vighnesh Shenoy --- tests/run-all.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/run-all.sh b/tests/run-all.sh index 49e6f861510..e4774a69262 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -2,8 +2,8 @@ set -u # TODO - Remove TypeScript regex after all tests have been migrated to Go. -E2E_REGEX_GO="${E2E_TEST_REGEX}_*test.go" -E2E_REGEX_TS="${E2E_TEST_REGEX}-*test.ts" +E2E_REGEX_GO=${E2E_TEST_REGEX:-*_test.go} +E2E_REGEX_TS=${E2E_TEST_REGEX:-*.test.ts} DIR=$(dirname "$0") cd $DIR @@ -31,6 +31,11 @@ function run_tests { continue fi + if [[ $test_case == setup_test.go || $test_case == cleanup_test.go ]]; + then + continue + fi + counter=$((counter+1)) # TODO - Remove condition after all tests have been migrated to Go. if [[ $test_case == *_test.go ]] From 15a3e8cad4295b8fcc60a943ef85d693f08606e6 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 18:15:18 +0530 Subject: [PATCH 13/15] Remove report.xml Signed-off-by: Vighnesh Shenoy --- report.xml | 307 ----------------------------------------------------- 1 file changed, 307 deletions(-) delete mode 100644 report.xml diff --git a/report.xml b/report.xml deleted file mode 100644 index b44ac0681d3..00000000000 --- a/report.xml +++ /dev/null @@ -1,307 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From c3dbb82c42178c25b99f4b205513653a6e9f3ede Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 10 Jun 2022 19:18:36 +0530 Subject: [PATCH 14/15] Exclude setup and cleanup files from runner scripts. Signed-off-by: Vighnesh Shenoy --- tests/run-all.sh | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/run-all.sh b/tests/run-all.sh index e4774a69262..d8e50f5e995 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -2,8 +2,8 @@ set -u # TODO - Remove TypeScript regex after all tests have been migrated to Go. -E2E_REGEX_GO=${E2E_TEST_REGEX:-*_test.go} -E2E_REGEX_TS=${E2E_TEST_REGEX:-*.test.ts} +E2E_REGEX_GO="./scalers*${E2E_TEST_REGEX:-*_test.go}" +E2E_REGEX_TS="./scalers*${E2E_TEST_REGEX:-*.test.ts}" DIR=$(dirname "$0") cd $DIR @@ -24,14 +24,9 @@ function run_tests { counter=0 # randomize tests order using shuf # TODO - Remove TypeScript regex after all tests have been migrated to Go. - for test_case in $(find . -name "$E2E_REGEX_GO" -o -name "$E2E_REGEX_TS" | shuf) + for test_case in $(find . -wholename "$E2E_REGEX_GO" -o -wholename "$E2E_REGEX_TS" | shuf) do - if [[ $test_case != *_test.go && $test_case != *.test.ts ]] # Skip helper files - then - continue - fi - - if [[ $test_case == setup_test.go || $test_case == cleanup_test.go ]]; + if [[ $test_case != *_test.go && $test_case != *.test.ts ]] # Skip helper files then continue fi From c9641e725aa92979fc60688828120a9e4fda62de Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Sat, 11 Jun 2022 17:01:42 +0530 Subject: [PATCH 15/15] Cleanup CRDs. Signed-off-by: Vighnesh Shenoy --- Makefile | 6 +++++- tests/clean-crds.sh | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100755 tests/clean-crds.sh diff --git a/Makefile b/Makefile index 7b796e30313..4d45e050020 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,10 @@ e2e-test-local: ## Run e2e tests against Kubernetes cluster configured in ~/.kub npm install --prefix tests ./tests/run-all.sh +.PHONY: e2e-test-clean-crds +e2e-test-clean-crds: ## Delete all scaled objects and jobs across all namespaces + ./tests/clean-crds.sh + .PHONY: e2e-test-clean e2e-test-clean: get-cluster-context ## Delete all namespaces labeled with type=e2e kubectl delete ns -l type=e2e @@ -251,7 +255,7 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in rm -rf config/default/kustomize-config/metadataLabelTransformer.yaml.out $(KUSTOMIZE) build config/default | kubectl apply -f - -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. +undeploy: e2e-test-clean-crds ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete -f - diff --git a/tests/clean-crds.sh b/tests/clean-crds.sh new file mode 100755 index 00000000000..bcaf925e09a --- /dev/null +++ b/tests/clean-crds.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +echo "Cleaning up scaled objects and jobs before undeploying KEDA" +while read -r namespace +do + resources=$(kubectl get so,sj -n $namespace -o name) + if [[ -n "$resources" ]] + then + kubectl delete $resources -n $namespace + fi +done < <(kubectl get namespaces -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}{end}")