diff --git a/tests/helper/helper.go b/tests/helper/helper.go index 0af757e6445..396047971a4 100644 --- a/tests/helper/helper.go +++ b/tests/helper/helper.go @@ -11,6 +11,7 @@ import ( "math/rand" "os" "os/exec" + "regexp" "strings" "testing" "text/template" @@ -426,3 +427,8 @@ func DeleteKubernetesResources(t *testing.T, kc *kubernetes.Clientset, nsName st func GetRandomNumber() int { return random.Intn(10000) } + +func RemoveANSI(input string) string { + reg := regexp.MustCompile(`(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]`) + return reg.ReplaceAllString(input, "") +} diff --git a/tests/scalers/postgresql-hashicorp-vault.test.ts b/tests/scalers/postgresql-hashicorp-vault.test.ts deleted file mode 100644 index 6420993f655..00000000000 --- a/tests/scalers/postgresql-hashicorp-vault.test.ts +++ /dev/null @@ -1,267 +0,0 @@ -import * as fs from 'fs' -import * as sh from 'shelljs' -import * as tmp from 'tmp' -import test from 'ava' -import { createNamespace, waitForDeploymentReplicaCount } from './helpers' - -const testNamespace = 'postgresql-hashicorp-vault' -const postgreSQLUsername = 'test-user' -const postgreSQLPassword = 'test-password' -const postgreSQLDatabase = 'test_db' -const deploymentName = 'worker' - -test.before(t => { - createNamespace(testNamespace) - - // install postgresql - const postgreSQLTmpFile = tmp.fileSync() - fs.writeFileSync(postgreSQLTmpFile.name, postgresqlDeploymentYaml.replace('{{POSTGRES_USER}}', postgreSQLUsername) - .replace('{{POSTGRES_PASSWORD}}', postgreSQLPassword) - .replace('{{POSTGRES_DB}}', postgreSQLDatabase) - .replace('{{POSTGRES_DB}}', postgreSQLDatabase)) - - t.is(0, sh.exec(`kubectl apply --namespace ${testNamespace} -f ${postgreSQLTmpFile.name}`).code, 'creating a POSTGRES deployment should work.') - // wait for postgresql to load - let postgresqlReadyReplicaCount = '0' - for (let i = 0; i < 30; i++) { - postgresqlReadyReplicaCount = sh.exec(`kubectl get deploy/postgresql -n ${testNamespace} -o jsonpath='{.status.readyReplicas}'`).stdout - if (postgresqlReadyReplicaCount != '1') { - sh.exec('sleep 2s') - } - } - t.is('1', postgresqlReadyReplicaCount, 'Postgresql is not in a ready state') - - // create table that used by the job and the worker - const postgresqlPod = sh.exec(`kubectl get po -n ${testNamespace} -o jsonpath='{.items[0].metadata.name}'`).stdout - t.not(postgresqlPod, '') - const createTableSQL = `CREATE TABLE task_instance (id serial PRIMARY KEY,state VARCHAR(10));` - sh.exec( `kubectl exec -n ${testNamespace} ${postgresqlPod} -- psql -U ${postgreSQLUsername} -d ${postgreSQLDatabase} -c "${createTableSQL}"`) - - // deploy hashicorp vault - sh.exec(`helm repo add hashicorp https://helm.releases.hashicorp.com`) - sh.exec(`helm repo update`) - let helmInstallStatus = sh.exec(`helm upgrade \ - --install \ - --set "server.dev.enabled=true" \ - --namespace ${testNamespace} \ - --wait \ - vault hashicorp/vault`).code - t.is(0, - helmInstallStatus, - 'deploying the Datadog Helm chart should work.' - ) - - // create a token and register the connection string - const connectionString = `postgresql://${postgreSQLUsername}:${postgreSQLPassword}@postgresql.${testNamespace}.svc.cluster.local:5432/${postgreSQLDatabase}?sslmode=disable` - let createSecret = sh.exec(`kubectl exec vault-0 --namespace ${testNamespace} -- vault kv put secret/keda connectionString=${connectionString}`).code - t.is(0, createSecret,'create secret in vault should work') - let response = JSON.parse(sh.exec(`kubectl exec vault-0 --namespace ${testNamespace} -- vault token create -format json`).stdout); - - sh.config.silent = true - // deploy streams consumer app, scaled object etc. - const tmpFile = tmp.fileSync() - const base64ConnectionString = Buffer.from(connectionString).toString('base64') - fs.writeFileSync(tmpFile.name, deployYaml.replace('{{HASHICORP_VAULT_TOKEN}}', response.auth.client_token) - .replace('{{POSTGRES_CONNECTION_STRING}}', base64ConnectionString) - .replace('{{DEPLOYMENT_NAME}}', deploymentName) - .replace('{{NAMESPACE}}', testNamespace)) - t.is( - 0, - sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${testNamespace}`).code, - 'creating a deployment should work..' - ) -}) - -test.serial('Deployment should have 0 replicas on start', async t => { - t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 60, 1000), 'replica count should start out as 0') - -}) - -test.serial(`Deployment should scale to 2 (the max) then back to 0`, async t => { - const tmpFile = tmp.fileSync() - fs.writeFileSync(tmpFile.name, insertRecordsJobYaml) - t.is( - 0, - sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${testNamespace}`).code, - 'creating job should work.' - ) - - const maxReplicaCount = 2 - t.true(await waitForDeploymentReplicaCount(maxReplicaCount, deploymentName, testNamespace, 120, 1000), 'Replica count should be 0 after 2 minutes') - - t.true(await waitForDeploymentReplicaCount(0, deploymentName, testNamespace, 360, 1000), 'Replica count should be 0 after 5 minutes') -}) - -test.after.always.cb('clean up postgresql deployment', t => { - const resources = [ - 'scaledobject.keda.sh/postgresql-scaledobject', - 'triggerauthentication.keda.sh/keda-trigger-hashicorp-vault-secret', - `deployment.apps/${deploymentName}`, - 'secret/postgresql-secrets', - 'job/postgresql-insert-job', - ] - - for (const resource of resources) { - sh.exec(`kubectl delete ${resource} --namespace ${testNamespace}`) - } - - // uninstall vault - sh.exec(`helm delete --namespace ${testNamespace} vault hashicorp/vault`) - - // uninstall postgresql - sh.exec(`kubectl delete --namespace ${testNamespace} deploy/postgresql`) - sh.exec(`kubectl delete namespace ${testNamespace}`) - - t.end() -}) - -const deployYaml = `apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: postgresql-update-worker - name: {{DEPLOYMENT_NAME}} -spec: - replicas: 0 - selector: - matchLabels: - app: postgresql-update-worker - template: - metadata: - labels: - app: postgresql-update-worker - spec: - containers: - - image: ghcr.io/kedacore/tests-postgresql - imagePullPolicy: Always - name: postgresql-processor-test - command: - - /app - - update - env: - - name: TASK_INSTANCES_COUNT - value: "6000" - - name: CONNECTION_STRING - valueFrom: - secretKeyRef: - name: postgresql-secrets - key: postgresql_conn_str ---- -apiVersion: v1 -kind: Secret -metadata: - name: postgresql-secrets -type: Opaque -data: - postgresql_conn_str: {{POSTGRES_CONNECTION_STRING}} ---- -apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: keda-trigger-hashicorp-vault-secret -spec: - hashiCorpVault: - address: http://vault.{{NAMESPACE}}:8200 - authentication: token - credential: - token: {{HASHICORP_VAULT_TOKEN}} - secrets: - - parameter: connection - key: connectionString - path: secret/data/keda ---- -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: postgresql-scaledobject -spec: - scaleTargetRef: - name: worker - pollingInterval: 5 - cooldownPeriod: 10 - minReplicaCount: 0 - maxReplicaCount: 2 - triggers: - - type: postgresql - metadata: - targetQueryValue: "4" - query: "SELECT CEIL(COUNT(*) / 5) FROM task_instance WHERE state='running' OR state='queued'" - authenticationRef: - name: keda-trigger-hashicorp-vault-secret` - -const insertRecordsJobYaml = `apiVersion: batch/v1 -kind: Job -metadata: - labels: - app: postgresql-insert-job - name: postgresql-insert-job -spec: - template: - metadata: - labels: - app: postgresql-insert-job - spec: - containers: - - image: ghcr.io/kedacore/tests-postgresql - imagePullPolicy: Always - name: postgresql-processor-test - command: - - /app - - insert - env: - - name: TASK_INSTANCES_COUNT - value: "6000" - - name: CONNECTION_STRING - valueFrom: - secretKeyRef: - name: postgresql-secrets - key: postgresql_conn_str - restartPolicy: Never - backoffLimit: 4` - - -const postgresqlDeploymentYaml = `apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: postgresql - name: postgresql -spec: - replicas: 1 - selector: - matchLabels: - app: postgresql - template: - metadata: - labels: - app: postgresql - spec: - containers: - - image: postgres:10.5 - name: postgresql - env: - - name: POSTGRES_USER - value: {{POSTGRES_USER}} - - name: POSTGRES_PASSWORD - value: {{POSTGRES_PASSWORD}} - - name: POSTGRES_DB - value: {{POSTGRES_DB}} - ports: - - name: postgresql - protocol: TCP - containerPort: 5432 ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: postgresql - name: postgresql -spec: - ports: - - port: 5432 - protocol: TCP - targetPort: 5432 - selector: - app: postgresql - type: ClusterIP` diff --git a/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go b/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go new file mode 100644 index 00000000000..35d5f345574 --- /dev/null +++ b/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go @@ -0,0 +1,390 @@ +//go:build e2e +// +build e2e + +package hashicorp_vault_test + +import ( + "encoding/base64" + "fmt" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "hashicorp-vault-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + vaultNamespace = "hashicorp-ns" + deploymentName = fmt.Sprintf("%s-deployment", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + triggerAuthenticationName = fmt.Sprintf("%s-ta", testName) + secretName = fmt.Sprintf("%s-secret", testName) + postgreSQLStatefulSetName = "postgresql" + postgresqlPodName = fmt.Sprintf("%s-0", postgreSQLStatefulSetName) + postgreSQLUsername = "test-user" + postgreSQLPassword = "test-password" + postgreSQLDatabase = "test_db" + postgreSQLConnectionString = fmt.Sprintf("postgresql://%s:%s@postgresql.%s.svc.cluster.local:5432/%s?sslmode=disable", + postgreSQLUsername, postgreSQLPassword, testNamespace, postgreSQLDatabase) + minReplicaCount = 0 + maxReplicaCount = 2 +) + +type templateData struct { + TestNamespace string + DeploymentName string + VaultNamespace string + ScaledObjectName string + TriggerAuthenticationName string + SecretName string + HashiCorpToken string + PostgreSQLStatefulSetName string + PostgreSQLConnectionStringBase64 string + PostgreSQLUsername string + PostgreSQLPassword string + PostgreSQLDatabase string + MinReplicaCount int + MaxReplicaCount int +} + +type templateValues map[string]string + +const ( + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: postgresql-update-worker + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} +spec: + replicas: 0 + selector: + matchLabels: + app: postgresql-update-worker + template: + metadata: + labels: + app: postgresql-update-worker + spec: + containers: + - image: ghcr.io/kedacore/tests-postgresql + imagePullPolicy: Always + name: postgresql-processor-test + command: + - /app + - update + env: + - name: TASK_INSTANCES_COUNT + value: "6000" + - name: CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: postgresql_conn_str +` + + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +type: Opaque +data: + postgresql_conn_str: {{.PostgreSQLConnectionStringBase64}} +` + + triggerAuthenticationTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthenticationName}} + namespace: {{.TestNamespace}} +spec: + hashiCorpVault: + address: http://vault.{{.VaultNamespace}}:8200 + authentication: token + credential: + token: {{.HashiCorpToken}} + secrets: + - parameter: connection + key: connectionString + path: secret/data/keda +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + cooldownPeriod: 10 + minReplicaCount: {{.MinReplicaCount}} + maxReplicaCount: {{.MaxReplicaCount}} + triggers: + - type: postgresql + metadata: + targetQueryValue: "4" + activationTargetQueryValue: "5" + query: "SELECT CEIL(COUNT(*) / 5) FROM task_instance WHERE state='running' OR state='queued'" + authenticationRef: + name: {{.TriggerAuthenticationName}} +` + + postgreSQLStatefulSetTemplate = ` +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: {{.PostgreSQLStatefulSetName}} + name: {{.PostgreSQLStatefulSetName}} + namespace: {{.TestNamespace}} +spec: + replicas: 1 + serviceName: {{.PostgreSQLStatefulSetName}} + selector: + matchLabels: + app: {{.PostgreSQLStatefulSetName}} + template: + metadata: + labels: + app: {{.PostgreSQLStatefulSetName}} + spec: + containers: + - image: postgres:10.5 + name: postgresql + env: + - name: POSTGRES_USER + value: {{.PostgreSQLUsername}} + - name: POSTGRES_PASSWORD + value: {{.PostgreSQLPassword}} + - name: POSTGRES_DB + value: {{.PostgreSQLDatabase}} + ports: + - name: postgresql + protocol: TCP + containerPort: 5432 +` + + postgreSQLServiceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{.PostgreSQLStatefulSetName}} + name: {{.PostgreSQLStatefulSetName}} + namespace: {{.TestNamespace}} +spec: + ports: + - port: 5432 + protocol: TCP + targetPort: 5432 + selector: + app: {{.PostgreSQLStatefulSetName}} + type: ClusterIP +` + + lowLevelRecordsJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: postgresql-insert-low-level-job + name: postgresql-insert-low-level-job + namespace: {{.TestNamespace}} +spec: + template: + metadata: + labels: + app: postgresql-insert-low-level-job + spec: + containers: + - image: ghcr.io/kedacore/tests-postgresql + imagePullPolicy: Always + name: postgresql-processor-test + command: + - /app + - insert + env: + - name: TASK_INSTANCES_COUNT + value: "20" + - name: CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: postgresql_conn_str + restartPolicy: Never + backoffLimit: 4 +` + + insertRecordsJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: postgresql-insert-job + name: postgresql-insert-job + namespace: {{.TestNamespace}} +spec: + template: + metadata: + labels: + app: postgresql-insert-job + spec: + containers: + - image: ghcr.io/kedacore/tests-postgresql + imagePullPolicy: Always + name: postgresql-processor-test + command: + - /app + - insert + env: + - name: TASK_INSTANCES_COUNT + value: "10000" + - name: CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: postgresql_conn_str + restartPolicy: Never + backoffLimit: 4 +` +) + +func TestPostreSQLScaler(t *testing.T) { + // Create kubernetes resources for PostgreSQL server + kc := GetKubernetesClient(t) + data, postgreSQLtemplates := getPostgreSQLTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, postgreSQLtemplates) + hashiCorpToken := setupHashiCorpVault(t, kc) + + assert.True(t, WaitForStatefulsetReplicaReadyCount(t, kc, postgreSQLStatefulSetName, testNamespace, 1, 60, 3), + "replica count should be %d after 3 minutes", 1) + + createTableSQL := "CREATE TABLE task_instance (id serial PRIMARY KEY,state VARCHAR(10));" + ok, out, errOut, err := WaitForSuccessfulExecCommandOnSpecificPod(t, postgresqlPodName, testNamespace, + fmt.Sprintf("psql -U %s -d %s -c \"%s\"", postgreSQLUsername, postgreSQLDatabase, createTableSQL), 60, 3) + assert.True(t, ok, "executing a command on PostreSQL Pod should work; Output: %s, ErrorOutput: %s, Error: %s", out, errOut, err) + + // Create kubernetes resources for testing + data, templates := getTemplateData() + data.HashiCorpToken = RemoveANSI(hashiCorpToken) + KubectlApplyMultipleWithTemplate(t, data, templates) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) + + testActivation(t, kc, data) + testScaleUp(t, kc, data) + testScaleDown(t, kc) + + // cleanup + KubectlDeleteMultipleWithTemplate(t, data, templates) + cleanupHashiCorpVault(t, kc) + DeleteKubernetesResources(t, kc, testNamespace, data, postgreSQLtemplates) +} + +func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset) string { + CreateNamespace(t, kc, vaultNamespace) + + _, err := ExecuteCommand("helm repo add hashicorp https://helm.releases.hashicorp.com") + assert.NoErrorf(t, err, "cannot add hashicorp repo - %s", err) + + _, err = ExecuteCommand("helm repo update") + assert.NoErrorf(t, err, "cannot update repos - %s", err) + + _, err = ExecuteCommand(fmt.Sprintf(`helm upgrade --install --set server.dev.enabled=true --namespace %s --wait vault hashicorp/vault`, vaultNamespace)) + assert.NoErrorf(t, err, "cannot install hashicorp vault - %s", err) + + podName := "vault-0" + + _, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault kv put secret/keda connectionString=%s", postgreSQLConnectionString)) + assert.NoErrorf(t, err, "cannot put connection string in hashicorp vault - %s", err) + + out, _, err := ExecCommandOnSpecificPod(t, podName, vaultNamespace, "vault token create -field token") + assert.NoErrorf(t, err, "cannot create hashicorp vault token - %s", err) + + return out +} + +func cleanupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset) { + _, err := ExecuteCommand(fmt.Sprintf("helm uninstall vault --namespace %s", vaultNamespace)) + assert.NoErrorf(t, err, "cannot uninstall hashicorp vault - %s", err) + + _, err = ExecuteCommand("helm repo remove hashicorp") + assert.NoErrorf(t, err, "cannot remove hashicorp repo - %s", err) + + DeleteNamespace(t, kc, vaultNamespace) +} + +func testActivation(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing activation ---") + templateTriggerJob := templateValues{"lowLevelRecordsJobTemplate": lowLevelRecordsJobTemplate} + KubectlApplyMultipleWithTemplate(t, data, templateTriggerJob) + + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, minReplicaCount, 60) +} + +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale up ---") + templateTriggerJob := templateValues{"insertRecordsJobTemplate": insertRecordsJobTemplate} + KubectlApplyMultipleWithTemplate(t, data, templateTriggerJob) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", maxReplicaCount) +} + +func testScaleDown(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale down ---") + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) +} + +var data = templateData{ + TestNamespace: testNamespace, + PostgreSQLStatefulSetName: postgreSQLStatefulSetName, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + MinReplicaCount: minReplicaCount, + MaxReplicaCount: maxReplicaCount, + TriggerAuthenticationName: triggerAuthenticationName, + SecretName: secretName, + PostgreSQLUsername: postgreSQLUsername, + PostgreSQLPassword: postgreSQLPassword, + PostgreSQLDatabase: postgreSQLDatabase, + PostgreSQLConnectionStringBase64: base64.StdEncoding.EncodeToString([]byte(postgreSQLConnectionString)), + VaultNamespace: vaultNamespace, +} + +func getPostgreSQLTemplateData() (templateData, map[string]string) { + return data, templateValues{ + "postgreSQLStatefulSetTemplate": postgreSQLStatefulSetTemplate, + "postgreSQLServiceTemplate": postgreSQLServiceTemplate, + } +} + +func getTemplateData() (templateData, map[string]string) { + return data, templateValues{ + "secretTemplate": secretTemplate, + "deploymentTemplate": deploymentTemplate, + "triggerAuthenticationTemplate": triggerAuthenticationTemplate, + "scaledObjectTemplate": scaledObjectTemplate, + } +}