Skip to content

Commit

Permalink
Add e2e test
Browse files Browse the repository at this point in the history 
Signed-off-by: Vighnesh Shenoy <vshenoy@microsoft.com>
  • Loading branch information
@v-shenoy
v-shenoy committed Oct 7, 2022
1 parent 24c5f97 commit dec8f51
Showing 1 changed file with 146 additions and 25 deletions.
171 changes: 146 additions & 25 deletions tests/internals/prometheus_metrics/prometheus_metrics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ var (
)

type templateData struct {
TestName string
TestNamespace string
DeploymentName string
ScaledObjectName string
Expand Down Expand Up @@ -176,6 +177,51 @@ spec:
targetPort: 8080
selector:
app: keda-operator
`

authenticationTemplate = `
apiVersion: v1
kind: Secret
metadata:
name: {{.TestName}}-secret
namespace: {{.TestNamespace}}
type: Opaque
stringData:
key: value
---
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: {{.TestName}}-ta1
namespace: {{.TestNamespace}}
spec:
secretTargetRef:
- parameter: param
name: {{.TestName}}-secret
key: key
---
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: {{.TestName}}-ta2
namespace: {{.TestNamespace}}
spec:
secretTargetRef:
- parameter: param
name: {{.TestName}}-secret
key: key
---
apiVersion: keda.sh/v1alpha1
kind: ClusterTriggerAuthentication
metadata:
name: {{.TestName}}-cta
namespace: {{.TestNamespace}}
spec:
secretTargetRef:
- parameter: param
name: {{.TestName}}-secret
key: key
---
`
)

Expand All @@ -194,14 +240,15 @@ func TestScaler(t *testing.T) {
"replica count should be 2 after 2 minute")

testHPAScalerMetricValue(t)
testTriggerTotalMetric(t, kc, data)
testOperatorMetrics(t, kc, data)

// cleanup
DeleteKubernetesResources(t, kc, testNamespace, data, templates)
}

func getTemplateData() (templateData, []Template) {
return templateData{
TestName: testName,
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
Expand All @@ -215,6 +262,7 @@ func getTemplateData() (templateData, []Template) {
{Name: "scaledObjectTemplate", Config: scaledObjectTemplate},
{Name: "clientTemplate", Config: clientTemplate},
{Name: "serviceTemplate", Config: serviceTemplate},
{Name: "authenticatioNTemplate", Config: authenticationTemplate},
}
}

Expand All @@ -225,10 +273,10 @@ func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*promMo
parser := expfmt.TextParser{}
// Ensure EOL
reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n"))
family, err := parser.TextToMetricFamilies(reader)
families, err := parser.TextToMetricFamilies(reader)
assert.NoErrorf(t, err, "cannot parse metrics - %s", err)

return family
return families
}

func testHPAScalerMetricValue(t *testing.T) {
Expand All @@ -254,29 +302,52 @@ func testHPAScalerMetricValue(t *testing.T) {
}
}

func testTriggerTotalMetric(t *testing.T, kc *kubernetes.Clientset, data templateData) {
t.Log("--- testing trigger total metric ---")
testTriggerTotalMetricValue(t, getTriggerTotalsManually(t, kc))
func testOperatorMetrics(t *testing.T, kc *kubernetes.Clientset, data templateData) {
t.Log("--- testing operator metrics ---")
testOperatorMetricValues(t, kc)

KubectlApplyWithTemplate(t, data, "cronScaledJobTemplate", cronScaledJobTemplate)
testTriggerTotalMetricValue(t, getTriggerTotalsManually(t, kc))
testOperatorMetricValues(t, kc)

KubectlDeleteWithTemplate(t, data, "cronScaledJobTemplate", cronScaledJobTemplate)
testTriggerTotalMetricValue(t, getTriggerTotalsManually(t, kc))
testOperatorMetricValues(t, kc)
}

func getTriggerTotalsManually(t *testing.T, kc *kubernetes.Clientset) map[string]int {
func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[string]int, map[string]map[string]int) {
kedaKc := GetKedaKubernetesClient(t)

triggerTotals := make(map[string]int)
crdTotals := map[string]map[string]int{
"scaled_object": {},
"scaled_job": {},
"trigger_authentication": {},
"cluster_trigger_authentication": {},
}

namespaceList, err := kc.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{})
assert.NoErrorf(t, err, "failed to list namespaces - %s", err)

clusterTriggerAuthenticationList, err := kedaKc.ClusterTriggerAuthentications().List(context.Background(), v1.ListOptions{})
assert.NoErrorf(t, err, "failed to list clusterTriggerAuthentications with err - %s")

for _, clusterTriggerAuth := range clusterTriggerAuthenticationList.Items {
namespace := clusterTriggerAuth.Namespace
if namespace == "" {
namespace = "default"
}
crdTotals["cluster_trigger_authentication"][namespace]++
}

for _, namespace := range namespaceList.Items {
namespaceName := namespace.Name
if namespace.Name == "" {
namespaceName = "default"
}

scaledObjectList, err := kedaKc.ScaledObjects(namespace.Name).List(context.Background(), v1.ListOptions{})
assert.NoErrorf(t, err, "failed to list scaledObjects in namespace - %s with err - %s", namespace.Name, err)

crdTotals["scaled_object"][namespaceName] = len(scaledObjectList.Items)
for _, scaledObject := range scaledObjectList.Items {
for _, trigger := range scaledObject.Spec.Triggers {
triggerTotals[trigger.Type]++
Expand All @@ -286,34 +357,84 @@ func getTriggerTotalsManually(t *testing.T, kc *kubernetes.Clientset) map[string
scaledJobList, err := kedaKc.ScaledJobs(namespace.Name).List(context.Background(), v1.ListOptions{})
assert.NoErrorf(t, err, "failed to list scaledJobs in namespace - %s with err - %s", namespace.Name, err)

crdTotals["scaled_job"][namespaceName] = len(scaledJobList.Items)
for _, scaledJob := range scaledJobList.Items {
for _, trigger := range scaledJob.Spec.Triggers {
triggerTotals[trigger.Type]++
}
}

triggerAuthList, err := kedaKc.TriggerAuthentications(namespace.Name).List(context.Background(), v1.ListOptions{})
assert.NoErrorf(t, err, "failed to list triggerAuthentications in namespace - %s with err - %s", namespace.Name, err)

crdTotals["trigger_authentication"][namespaceName] = len(triggerAuthList.Items)
}

return triggerTotals
return triggerTotals, crdTotals
}

func testTriggerTotalMetricValue(t *testing.T, expected map[string]int) {
family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure http://%s.keda:8080/metrics", serviceName))
func testOperatorMetricValues(t *testing.T, kc *kubernetes.Clientset) {
families := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure http://%s.keda:8080/metrics", serviceName))
expectedTriggerTotals, expectedCrdTotals := getOperatorMetricsManually(t, kc)

if val, ok := family["keda_operator_trigger_totals"]; ok {
var found bool
metrics := val.GetMetric()
for _, metric := range metrics {
labels := metric.GetLabel()
for _, label := range labels {
if *label.Name == "type" {
assert.Equalf(t, float64(expected[*label.Value]), *metric.Gauge.Value, "expected %f got %f for type %s",
float64(expected[*label.Value]), *metric.Gauge.Value, *label.Value)
found = true
}
checkTriggerTotalValues(t, families, expectedTriggerTotals)
checkCRDTotalValues(t, families, expectedCrdTotals)
}

func checkTriggerTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]int) {
t.Log("--- testing trigger total metrics ---")

family, ok := families["keda_operator_trigger_totals"]
if !ok {
t.Errorf("metric not available")
return
}

metrics := family.GetMetric()
for _, metric := range metrics {
labels := metric.GetLabel()
for _, label := range labels {
if *label.Name == "type" {
triggerType := *label.Value
metricValue := *metric.Gauge.Value
expectedMetricValue := float64(expected[triggerType])

assert.Equalf(t, expectedMetricValue, metricValue, "expected %f got %f for trigger type %s",
expectedMetricValue, metricValue, triggerType)

delete(expected, triggerType)
}
}
assert.Equal(t, true, found)
} else {
}

assert.Equal(t, 0, len(expected))
}

func checkCRDTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]map[string]int) {
t.Log("--- testing crd total metrics ---")

family, ok := families["keda_operator_crd_totals"]
if !ok {
t.Errorf("metric not available")
return
}

metrics := family.GetMetric()
for _, metric := range metrics {
labels := metric.GetLabel()
var namespace, crdType string
for _, label := range labels {
if *label.Name == "type" {
crdType = *label.Value
} else if *label.Name == "namespace" {
namespace = *label.Value
}
}

metricValue := *metric.Gauge.Value
expectedMetricValue := float64(expected[crdType][namespace])

assert.Equalf(t, expectedMetricValue, metricValue, "expected %f got %f for crd type %s & namespace %s",
expectedMetricValue, metricValue, crdType, namespace)
}
}

0 comments on commit dec8f51

Please sign in to comment.