Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add E2E tests for RabbitMQ with Azure Workload Identity auth & fix ra…
Browse files Browse the repository at this point in the history
…bbitmq generic tests

Signed-off-by: Jakub Adamus <jakub.admaus@vivantis.cz>
Jakub Adamus authored and Jakub Adamus committed Jun 15, 2023
1 parent 62a51bf commit 0368ebd
Showing 9 changed files with 421 additions and 28 deletions.
63 changes: 53 additions & 10 deletions tests/scalers/rabbitmq/rabbitmq_helper.go
Original file line number Diff line number Diff line change
@@ -60,6 +60,15 @@ data:
default_vhost = {{.VHostName}}
management.tcp.port = 15672
management.tcp.ip = 0.0.0.0
{{if .EnableOAuth}}
auth_backends.1 = rabbit_auth_backend_internal
auth_backends.2 = rabbit_auth_backend_oauth2
auth_backends.3 = rabbit_auth_backend_amqp
auth_oauth2.resource_server_id = {{.OAuthClientId}}
auth_oauth2.scope_prefix = rabbitmq.
auth_oauth2.additional_scopes_key = {{.OAuthScopesKey}}
auth_oauth2.jwks_url = {{.OAuthJwksUri}}
{{end}}
enabled_plugins: |
[rabbitmq_management].
---
@@ -158,33 +167,67 @@ spec:
`
)

type RabbitOAuthConfig struct {
Enable bool
ClientId string
ScopesKey string
JwksUrl string
}

func WithoutOAuth() RabbitOAuthConfig {
return RabbitOAuthConfig{
Enable: false,
}
}

func WithAzureADOAuth(tenantId string, clientId string) RabbitOAuthConfig {
return RabbitOAuthConfig{
Enable: true,
ClientId: clientId,
ScopesKey: "roles",
JwksUrl: fmt.Sprintf("https://login.microsoftonline.com/%s/discovery/keys", tenantId),
}
}

type templateData struct {
Namespace string
Connection string
QueueName string
HostName, VHostName string
Username, Password string
MessageCount int
EnableOAuth bool
OAuthClientId string
OAuthScopesKey string
OAuthJwksUri string
}

func RMQInstall(t *testing.T, kc *kubernetes.Clientset, namespace, user, password, vhost string) {
func RMQInstall(t *testing.T, kc *kubernetes.Clientset, namespace, user, password, vhost string, oauth RabbitOAuthConfig) {
helper.CreateNamespace(t, kc, namespace)
data := templateData{
Namespace: namespace,
VHostName: vhost,
Username: user,
Password: password,
Namespace: namespace,
VHostName: vhost,
Username: user,
Password: password,
EnableOAuth: oauth.Enable,
OAuthClientId: oauth.ClientId,
OAuthScopesKey: oauth.ScopesKey,
OAuthJwksUri: oauth.JwksUrl,
}

helper.KubectlApplyWithTemplate(t, data, "rmqDeploymentTemplate", deploymentTemplate)
}

func RMQUninstall(t *testing.T, namespace, user, password, vhost string) {
func RMQUninstall(t *testing.T, namespace, user, password, vhost string, oauth RabbitOAuthConfig) {
data := templateData{
Namespace: namespace,
VHostName: vhost,
Username: user,
Password: password,
Namespace: namespace,
VHostName: vhost,
Username: user,
Password: password,
EnableOAuth: oauth.Enable,
OAuthClientId: oauth.ClientId,
OAuthScopesKey: oauth.ScopesKey,
OAuthJwksUri: oauth.JwksUrl,
}

helper.KubectlDeleteWithTemplate(t, data, "rmqDeploymentTemplate", deploymentTemplate)
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import (
)

// Load environment variables from .env file
var _ = godotenv.Load("../../.env")
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-amqp-test"
@@ -79,7 +79,7 @@ func TestScaler(t *testing.T) {
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost)
RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithoutOAuth())
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
@@ -92,7 +92,7 @@ func TestScaler(t *testing.T) {
// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithoutOAuth())
}

func getTemplateData() (templateData, []Template) {
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import (
)

// Load environment variables from .env file
var _ = godotenv.Load("../../.env")
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-amqp-vhost-test"
@@ -79,7 +79,7 @@ func TestScaler(t *testing.T) {
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost)
RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithoutOAuth())
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
@@ -92,7 +92,7 @@ func TestScaler(t *testing.T) {
// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithoutOAuth())
}

func getTemplateData() (templateData, []Template) {
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import (
)

// Load environment variables from .env file
var _ = godotenv.Load("../../.env")
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-http-test"
@@ -80,7 +80,7 @@ func TestScaler(t *testing.T) {
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost)
RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithoutOAuth())
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
@@ -91,7 +91,7 @@ func TestScaler(t *testing.T) {
// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithoutOAuth())
}

func getTemplateData() (templateData, []Template) {
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
//go:build e2e
// +build e2e

package rabbitmq_queue_http_add_wi_test

import (
"encoding/base64"
"fmt"
"github.com/stretchr/testify/require"
"os"
"testing"

"github.com/joho/godotenv"
"github.com/stretchr/testify/assert"
"k8s.io/client-go/kubernetes"

. "github.com/kedacore/keda/v2/tests/helper"
. "github.com/kedacore/keda/v2/tests/scalers/rabbitmq"
)

// Load environment variables from .env file
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-http-test-aad-wi"
)

var (
testNamespace = fmt.Sprintf("%s-ns", testName)
rmqNamespace = fmt.Sprintf("%s-rmq", testName)
deploymentName = fmt.Sprintf("%s-deployment", testName)
secretName = fmt.Sprintf("%s-secret", testName)
triggerAuthName = fmt.Sprintf("%s-ta", testName)
triggerSecretName = fmt.Sprintf("%s-ta-secret", testName)
scaledObjectName = fmt.Sprintf("%s-so", testName)
queueName = "hello"
user = fmt.Sprintf("%s-user", testName)
password = fmt.Sprintf("%s-password", testName)
vhost = "/"
connectionString = fmt.Sprintf("amqp://%s:%s@rabbitmq.%s.svc.cluster.local/", user, password, rmqNamespace)
httpConnectionString = fmt.Sprintf("http://%s:%s@rabbitmq.%s.svc.cluster.local/", user, password, rmqNamespace)
httpNoAuthConnectionString = fmt.Sprintf("http://rabbitmq.%s.svc.cluster.local/", rmqNamespace)
rabbitAppClientID = os.Getenv("TF_AZURE_RABBIT_API_APPLICATION_ID")
azureADTenantID = os.Getenv("TF_AZURE_SP_TENANT")
messageCount = 100
)

const (
secretTemplate = `
apiVersion: v1
kind: Secret
metadata:
name: {{.TriggerSecretName}}
namespace: {{.TestNamespace}}
type: Opaque
data:
workloadIdentityResource: {{.Base64RabbitAppClientId}}
`

triggerAuthTemplate = `
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: {{.TriggerAuthName}}
namespace: {{.TestNamespace}}
spec:
podIdentity:
provider: azure-workload
secretTargetRef:
- parameter: workloadIdentityResource
name: {{.TriggerSecretName}}
key: workloadIdentityResource
`
scaledObjectTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.ScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 5
cooldownPeriod: 10
minReplicaCount: 0
maxReplicaCount: 4
triggers:
- type: rabbitmq
metadata:
queueName: {{.QueueName}}
vhostName: {{.VHost}}
host: {{.ConnectionNoAuth}}
protocol: http
mode: QueueLength
value: '10'
authenticationRef:
name: {{.TriggerAuthName}}
`
)

type templateData struct {
TestNamespace string
DeploymentName string
TriggerSecretName string
TriggerAuthName string
ScaledObjectName string
SecretName string
QueueName string
VHost string
Connection, Base64Connection string
ConnectionNoAuth string
RabbitAppClientId, Base64RabbitAppClientId string
}

func TestScaler(t *testing.T) {
// setup
t.Log("--- setting up ---")
require.NotEmpty(t, rabbitAppClientID, "TF_AZURE_RABBIT_API_APPLICATION_ID env variable is required for rabbitmq workload identity tests")
require.NotEmpty(t, azureADTenantID, "TF_AZURE_SP_TENANT env variable is required for rabbitmq workload identity tests")

// Create kubernetes resources
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithAzureADOAuth(azureADTenantID, rabbitAppClientID))
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
"replica count should be 0 after 1 minute")

testScaling(t, kc)

// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithAzureADOAuth(azureADTenantID, rabbitAppClientID))
}

func getTemplateData() (templateData, []Template) {
return templateData{
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
SecretName: secretName,
VHost: vhost,
QueueName: queueName,
Connection: connectionString,
Base64Connection: base64.StdEncoding.EncodeToString([]byte(httpConnectionString)),
TriggerAuthName: triggerAuthName,
TriggerSecretName: triggerSecretName,
ConnectionNoAuth: httpNoAuthConnectionString,
RabbitAppClientId: rabbitAppClientID,
Base64RabbitAppClientId: base64.StdEncoding.EncodeToString([]byte(rabbitAppClientID)),
}, []Template{
{Name: "deploymentTemplate", Config: RMQTargetDeploymentTemplate},
{Name: "secretTemplate", Config: secretTemplate},
{Name: "triggerAuthTemplate", Config: triggerAuthTemplate},
{Name: "scaledObjectTemplate", Config: scaledObjectTemplate},
}
}

func testScaling(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale out ---")
RMQPublishMessages(t, rmqNamespace, connectionString, queueName, messageCount)
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 4, 60, 1),
"replica count should be 4 after 1 minute")

t.Log("--- testing scale in ---")
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
"replica count should be 0 after 1 minute")
}
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import (
)

// Load environment variables from .env file
var _ = godotenv.Load("../../.env")
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-http-regex-test"
@@ -83,7 +83,7 @@ func TestScaler(t *testing.T) {
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost)
RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithoutOAuth())
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
@@ -94,7 +94,7 @@ func TestScaler(t *testing.T) {
// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithoutOAuth())
}

func getTemplateData() (templateData, []Template) {
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
//go:build e2e
// +build e2e

package rabbitmq_queue_http_regex_aad_wi_test

import (
"encoding/base64"
"fmt"
"github.com/stretchr/testify/require"
"os"
"testing"

"github.com/joho/godotenv"
"github.com/stretchr/testify/assert"
"k8s.io/client-go/kubernetes"

. "github.com/kedacore/keda/v2/tests/helper"
. "github.com/kedacore/keda/v2/tests/scalers/rabbitmq"
)

// Load environment variables from .env file
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-http-regex-aad-wi-test"
)

var (
testNamespace = fmt.Sprintf("%s-ns", testName)
rmqNamespace = fmt.Sprintf("%s-rmq", testName)
deploymentName = fmt.Sprintf("%s-deployment", testName)
secretName = fmt.Sprintf("%s-secret", testName)
triggerAuthName = fmt.Sprintf("%s-ta", testName)
triggerSecretName = fmt.Sprintf("%s-ta-secret", testName)
scaledObjectName = fmt.Sprintf("%s-so", testName)
queueName = "hello"
queueRegex = "^hell.{1}$"
user = fmt.Sprintf("%s-user", testName)
password = fmt.Sprintf("%s-password", testName)
vhost = "/"
connectionString = fmt.Sprintf("amqp://%s:%s@rabbitmq.%s.svc.cluster.local/", user, password, rmqNamespace)
httpConnectionString = fmt.Sprintf("http://%s:%s@rabbitmq.%s.svc.cluster.local/", user, password, rmqNamespace)
httpNoAuthConnectionString = fmt.Sprintf("http://rabbitmq.%s.svc.cluster.local/", rmqNamespace)
rabbitAppClientID = os.Getenv("TF_AZURE_RABBIT_API_APPLICATION_ID")
azureADTenantID = os.Getenv("TF_AZURE_SP_TENANT")
messageCount = 100
)

const (
secretTemplate = `
apiVersion: v1
kind: Secret
metadata:
name: {{.TriggerSecretName}}
namespace: {{.TestNamespace}}
type: Opaque
data:
workloadIdentityResource: {{.Base64RabbitAppClientId}}
`
triggerAuthTemplate = `
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: {{.TriggerAuthName}}
namespace: {{.TestNamespace}}
spec:
podIdentity:
provider: azure-workload
secretTargetRef:
- parameter: workloadIdentityResource
name: {{.TriggerSecretName}}
key: workloadIdentityResource
`
scaledObjectTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.ScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 5
cooldownPeriod: 10
minReplicaCount: 0
maxReplicaCount: 4
triggers:
- type: rabbitmq
metadata:
queueName: {{.QueueName}}
vhostName: {{.VHost}}
host: {{.ConnectionNoAuth}}
protocol: http
mode: QueueLength
value: '10'
useRegex: 'true'
operation: sum
authenticationRef:
name: {{.TriggerAuthName}}
`
)

type templateData struct {
TestNamespace string
DeploymentName string
TriggerSecretName string
TriggerAuthName string
ScaledObjectName string
SecretName string
QueueName string
VHost string
Connection, Base64Connection string
ConnectionNoAuth string
RabbitAppClientId, Base64RabbitAppClientId string
}

func TestScaler(t *testing.T) {
// setup
t.Log("--- setting up ---")
require.NotEmpty(t, rabbitAppClientID, "TF_AZURE_RABBIT_API_APPLICATION_ID env variable is required for rabbitmq workload identity tests")
require.NotEmpty(t, azureADTenantID, "TF_AZURE_SP_TENANT env variable is required for rabbitmq workload identity tests")

// Create kubernetes resources
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithAzureADOAuth(azureADTenantID, rabbitAppClientID))
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
"replica count should be 0 after 1 minute")

testScaling(t, kc)

// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithAzureADOAuth(azureADTenantID, rabbitAppClientID))
}

func getTemplateData() (templateData, []Template) {
return templateData{
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
SecretName: secretName,
VHost: vhost,
QueueName: queueRegex,
Connection: connectionString,
Base64Connection: base64.StdEncoding.EncodeToString([]byte(httpConnectionString)),
TriggerAuthName: triggerAuthName,
TriggerSecretName: triggerSecretName,
ConnectionNoAuth: httpNoAuthConnectionString,
RabbitAppClientId: rabbitAppClientID,
Base64RabbitAppClientId: base64.StdEncoding.EncodeToString([]byte(rabbitAppClientID)),
}, []Template{
{Name: "deploymentTemplate", Config: RMQTargetDeploymentTemplate},
{Name: "secretTemplate", Config: secretTemplate},
{Name: "triggerAuthTemplate", Config: triggerAuthTemplate},
{Name: "scaledObjectTemplate", Config: scaledObjectTemplate},
}
}

func testScaling(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale out ---")
RMQPublishMessages(t, rmqNamespace, connectionString, queueName, messageCount)
// dummies
RMQPublishMessages(t, rmqNamespace, connectionString, fmt.Sprintf("%s-1", queueName), messageCount)
RMQPublishMessages(t, rmqNamespace, connectionString, fmt.Sprintf("%s-%s", queueName, queueName), messageCount)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 4, 60, 2),
"replica count should be 4 after 2 minute")

t.Log("--- testing scale in ---")
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
"replica count should be 0 after 1 minute")
}
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import (
)

// Load environment variables from .env file
var _ = godotenv.Load("../../.env")
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-http-regex-vhost-test"
@@ -88,7 +88,7 @@ func TestScaler(t *testing.T) {
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost)
RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithoutOAuth())
CreateKubernetesResources(t, kc, testNamespace, data, templates)

RMQCreateVHost(t, rmqNamespace, connectionHost, user, password, dummyVhost1)
@@ -102,7 +102,7 @@ func TestScaler(t *testing.T) {
// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithoutOAuth())
}

func getTemplateData() (templateData, []Template) {
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import (
)

// Load environment variables from .env file
var _ = godotenv.Load("../../.env")
var _ = godotenv.Load("../../../.env")

const (
testName = "rmq-queue-http-vhost-test"
@@ -80,7 +80,7 @@ func TestScaler(t *testing.T) {
kc := GetKubernetesClient(t)
data, templates := getTemplateData()

RMQInstall(t, kc, rmqNamespace, user, password, vhost)
RMQInstall(t, kc, rmqNamespace, user, password, vhost, WithoutOAuth())
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1),
@@ -91,7 +91,7 @@ func TestScaler(t *testing.T) {
// cleanup
t.Log("--- cleaning up ---")
DeleteKubernetesResources(t, testNamespace, data, templates)
RMQUninstall(t, rmqNamespace, user, password, vhost)
RMQUninstall(t, rmqNamespace, user, password, vhost, WithoutOAuth())
}

func getTemplateData() (templateData, []Template) {

0 comments on commit 0368ebd

Please sign in to comment.