From 14e9c7f46685ab0932c3342529282992a40654d1 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 2 Apr 2020 08:59:01 +0300 Subject: [PATCH] Add e2e tests for ingress A/B Testing --- pkg/router/ingress.go | 9 +- pkg/router/ingress_test.go | 3 - test/e2e-nginx-tests.sh | 200 ++++++++++++++++++++----------------- test/e2e-nginx.sh | 2 +- 4 files changed, 110 insertions(+), 104 deletions(-) diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index bf77eb2ad..928fce7a7 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -166,11 +166,8 @@ func (i *IngressRouter) SetRoutes( cookie = v.Exact } else { header = k - if v.Regex != "" { - headerRegex = v.Regex - } else { - headerValue = v.Exact - } + headerRegex = v.Regex + headerValue = v.Exact } } } @@ -206,7 +203,6 @@ func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[strin } res[i.GetAnnotationWithPrefix("canary")] = "false" - res[i.GetAnnotationWithPrefix("canary-weight")] = "0" return res } @@ -221,7 +217,6 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, } res[i.GetAnnotationWithPrefix("canary")] = "true" - res[i.GetAnnotationWithPrefix("canary-weight")] = "0" if cookie != "" { res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go index dbec77b8c..b1fe5f388 100644 --- a/pkg/router/ingress_test.go +++ b/pkg/router/ingress_test.go @@ -25,7 +25,6 @@ func TestIngressRouter_Reconcile(t *testing.T) { require.NoError(t, err) canaryAn := "custom.ingress.kubernetes.io/canary" - canaryWeightAn := "custom.ingress.kubernetes.io/canary-weight" canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) @@ -33,7 +32,6 @@ func TestIngressRouter_Reconcile(t *testing.T) { // test initialisation assert.Equal(t, "false", inCanary.Annotations[canaryAn]) - assert.Equal(t, "0", inCanary.Annotations[canaryWeightAn]) } func TestIngressRouter_GetSetRoutes(t *testing.T) { @@ -80,7 +78,6 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) { // test promotion assert.Equal(t, "false", inCanary.Annotations[canaryAn]) - assert.Equal(t, "0", inCanary.Annotations[canaryWeightAn]) } func TestIngressRouter_ABTest(t *testing.T) { diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh index 462d2016c..fe77002db 100755 --- a/test/e2e-nginx-tests.sh +++ b/test/e2e-nginx-tests.sh @@ -18,6 +18,65 @@ echo '>>> Initialising canary' kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml +echo '>>> Create metric templates' +cat <>> Waiting for primary to be ready' @@ -173,73 +204,56 @@ spec: progressDeadlineSeconds: 60 service: port: 80 - targetPort: 9898 + targetPort: http analysis: - interval: 10s + interval: 15s threshold: 5 - iterations: 5 + iterations: 3 match: - - headers: - x-canary: - exact: "insider" - - headers: - cookie: - exact: "canary" + - headers: + x-user: + exact: "insider" metrics: - - name: "http-request-success-rate" - threshold: 99 - interval: 1m - query: | - 100 - sum( - rate( - http_request_duration_seconds_count{ - kubernetes_namespace="test", - kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", - path="root", - status!~"5.*" - }[1m] - ) - ) - / - sum( - rate( - http_request_duration_seconds_count{ - kubernetes_namespace="test", - kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", - path="root" - }[1m] - ) - ) - * 100 + - name: error-rate + templateRef: + name: error-rate + namespace: ingress-nginx + thresholdRange: + max: 1 + interval: 30s + - name: latency + templateRef: + name: latency + namespace: ingress-nginx + thresholdRange: + max: 0.5 + interval: 30s webhooks: - - name: pre - type: pre-rollout + - name: test-header-routing + type: rollout url: http://flagger-loadtester.test/ timeout: 5s metadata: - type: cmd - cmd: "hey -z 10m -q 10 -c 2 -H 'X-Canary: insider' -host app.example.com http://nginx-ingress-controller.ingress-nginx" - logCmdOutput: "true" - - name: post - type: post-rollout + type: bash + cmd: "curl -sH 'x-user: insider' -H 'Host: app.example.com' http://nginx-ingress-controller.ingress-nginx | grep '3.1.2'" + - name: load-test + type: rollout url: http://flagger-loadtester.test/ - timeout: 15s metadata: type: cmd - cmd: "curl -sH 'Host: app.example.com' http://nginx-ingress-controller.ingress-nginx" - logCmdOutput: "true" + cmd: "hey -z 2m -q 10 -c 2 -H 'x-user: insider' -host app.example.com http://nginx-ingress-controller.ingress-nginx" EOF echo '>>> Triggering A/B testing' kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 echo '>>> Waiting for A/B testing promotion' -retries=50 +retries=6 count=0 ok=false until ${ok}; do kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false - sleep 10 + sleep 30 kubectl -n ingress-nginx logs deployment/flagger --tail 1 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then @@ -255,4 +269,4 @@ echo '✔ A/B testing promotion test passed' kubectl -n ingress-nginx logs deployment/flagger -echo '✔ All tests passed' \ No newline at end of file +echo '✔ All tests passed' diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh index eb7b8e8df..c0f4ee95e 100755 --- a/test/e2e-nginx.sh +++ b/test/e2e-nginx.sh @@ -3,7 +3,7 @@ set -o errexit REPO_ROOT=$(git rev-parse --show-toplevel) -NGINX_HELM_VERSION=1.34.2 # ingress v0.30.0 +NGINX_HELM_VERSION=1.34.3 # ingress v0.30.0 echo '>>> Installing NGINX Ingress' kubectl create ns ingress-nginx