Add e2e tests for ingress A/B Testing

fluxcd · Apr 2, 2020 · 14e9c7f · 14e9c7f
1 parent b8e9f57
commit 14e9c7f
Show file tree

Hide file tree

Showing 4 changed files with 110 additions and 104 deletions.
diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go
@@ -166,11 +166,8 @@ func (i *IngressRouter) SetRoutes(
  cookie = v.Exact
  } else {
  header = k
- if v.Regex != "" {
- headerRegex = v.Regex
- } else {
- headerValue = v.Exact
- }
+ headerRegex = v.Regex
+ headerValue = v.Exact
  }
  }
  }
@@ -206,7 +203,6 @@ func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[strin
  }
 
  res[i.GetAnnotationWithPrefix("canary")] = "false"
- res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
  return res
 }
@@ -221,7 +217,6 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string,
  }
 
  res[i.GetAnnotationWithPrefix("canary")] = "true"
- res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
  if cookie != "" {
  res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie

diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go
@@ -25,15 +25,13 @@ func TestIngressRouter_Reconcile(t *testing.T) {
  require.NoError(t, err)
 
  canaryAn := "custom.ingress.kubernetes.io/canary"
- canaryWeightAn := "custom.ingress.kubernetes.io/canary-weight"
 
  canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
  inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{})
  require.NoError(t, err)
 
  // test initialisation
  assert.Equal(t, "false", inCanary.Annotations[canaryAn])
- assert.Equal(t, "0", inCanary.Annotations[canaryWeightAn])
 }
 
 func TestIngressRouter_GetSetRoutes(t *testing.T) {
@@ -80,7 +78,6 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
 
  // test promotion
  assert.Equal(t, "false", inCanary.Annotations[canaryAn])
- assert.Equal(t, "0", inCanary.Annotations[canaryWeightAn])
 }
 
 func TestIngressRouter_ABTest(t *testing.T) {

diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh
@@ -18,6 +18,65 @@ echo '>>> Initialising canary'
 kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
 kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml
 
+echo '>>> Create metric templates'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+ name: error-rate
+ namespace: ingress-nginx
+spec:
+ provider:
+ type: prometheus
+ address: http://flagger-prometheus.ingress-nginx:9090
+ query: |
+ 100 - sum(
+ rate(
+ http_request_duration_seconds_count{
+ kubernetes_namespace="{{ namespace }}",
+ kubernetes_pod_name=~"{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+ path="root",
+ status!~"5.*"
+ }[{{ interval }}]
+ )
+ )
+ /
+ sum(
+ rate(
+ http_request_duration_seconds_count{
+ kubernetes_namespace="{{ namespace }}",
+ kubernetes_pod_name=~"{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+ path="root"
+ }[{{ interval }}]
+ )
+ )
+ * 100
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+ name: latency
+ namespace: ingress-nginx
+spec:
+ provider:
+ type: prometheus
+ address: http://flagger-prometheus.ingress-nginx:9090
+ query: |
+ histogram_quantile(0.99,
+ sum(
+ rate(
+ http_request_duration_seconds_bucket{
+ kubernetes_namespace="{{ namespace }}",
+ kubernetes_pod_name=~"{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+ path="root"
+ }[{{ interval }}]
+ )
+ ) by (le)
+ )
+EOF
+
 cat <<EOF | kubectl apply -f -
 apiVersion: flagger.app/v1beta1
 kind: Canary
@@ -39,58 +98,30 @@ spec:
  targetPort: http
  analysis:
  interval: 15s
- threshold: 15
- maxWeight: 30
- stepWeight: 10
+ threshold: 5
+ maxWeight: 40
+ stepWeight: 20
  metrics:
- - name: "http-request-success-rate"
- threshold: 99
- interval: 1m
- query: |
- 100 - sum(
- rate(
- http_request_duration_seconds_count{
- kubernetes_namespace="test",
- kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
- path="root",
- status!~"5.*"
- }[1m]
- )
- )
- /
- sum(
- rate(
- http_request_duration_seconds_count{
- kubernetes_namespace="test",
- kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
- path="root"
- }[1m]
- )
- )
- * 100
- - name: "latency"
- threshold: 0.5
- interval: 1m
- query: |
- histogram_quantile(0.99,
- sum(
- rate(
- http_request_duration_seconds_bucket{
- kubernetes_namespace="test",
- kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
- path="root"
- }[1m]
- )
- ) by (le)
- )
+ - name: error-rate
+ templateRef:
+ name: error-rate
+ namespace: ingress-nginx
+ thresholdRange:
+ max: 1
+ interval: 30s
+ - name: latency
+ templateRef:
+ name: latency
+ namespace: ingress-nginx
+ thresholdRange:
+ max: 0.5
+ interval: 30s
  webhooks:
  - name: load-test
  url: http://flagger-loadtester.test/
- timeout: 5s
  metadata:
  type: cmd
- cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://nginx-ingress-controller.ingress-nginx"
- logCmdOutput: "true"
+ cmd: "hey -z 2m -q 10 -c 2 -host app.example.com http://nginx-ingress-controller.ingress-nginx"
 EOF
 
 echo '>>> Waiting for primary to be ready'
@@ -173,73 +204,56 @@ spec:
  progressDeadlineSeconds: 60
  service:
  port: 80
- targetPort: 9898
+ targetPort: http
  analysis:
- interval: 10s
+ interval: 15s
  threshold: 5
- iterations: 5
+ iterations: 3
  match:
- - headers:
- x-canary:
- exact: "insider"
- - headers:
- cookie:
- exact: "canary"
+ - headers:
+ x-user:
+ exact: "insider"
  metrics:
- - name: "http-request-success-rate"
- threshold: 99
- interval: 1m
- query: |
- 100 - sum(
- rate(
- http_request_duration_seconds_count{
- kubernetes_namespace="test",
- kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
- path="root",
- status!~"5.*"
- }[1m]
- )
- )
- /
- sum(
- rate(
- http_request_duration_seconds_count{
- kubernetes_namespace="test",
- kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
- path="root"
- }[1m]
- )
- )
- * 100
+ - name: error-rate
+ templateRef:
+ name: error-rate
+ namespace: ingress-nginx
+ thresholdRange:
+ max: 1
+ interval: 30s
+ - name: latency
+ templateRef:
+ name: latency
+ namespace: ingress-nginx
+ thresholdRange:
+ max: 0.5
+ interval: 30s
  webhooks:
- - name: pre
- type: pre-rollout
+ - name: test-header-routing
+ type: rollout
  url: http://flagger-loadtester.test/
  timeout: 5s
  metadata:
- type: cmd
- cmd: "hey -z 10m -q 10 -c 2 -H 'X-Canary: insider' -host app.example.com http://nginx-ingress-controller.ingress-nginx"
- logCmdOutput: "true"
- - name: post
- type: post-rollout
+ type: bash
+ cmd: "curl -sH 'x-user: insider' -H 'Host: app.example.com' http://nginx-ingress-controller.ingress-nginx | grep '3.1.2'"
+ - name: load-test
+ type: rollout
  url: http://flagger-loadtester.test/
- timeout: 15s
  metadata:
  type: cmd
- cmd: "curl -sH 'Host: app.example.com' http://nginx-ingress-controller.ingress-nginx"
- logCmdOutput: "true"
+ cmd: "hey -z 2m -q 10 -c 2 -H 'x-user: insider' -host app.example.com http://nginx-ingress-controller.ingress-nginx"
 EOF
 
 echo '>>> Triggering A/B testing'
 kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2
 
 echo '>>> Waiting for A/B testing promotion'
-retries=50
+retries=6
 count=0
 ok=false
 until ${ok}; do
  kubectl -n test describe deployment/podinfo-primary | grep '3.1.2' && ok=true || ok=false
- sleep 10
+ sleep 30
  kubectl -n ingress-nginx logs deployment/flagger --tail 1
  count=$(($count + 1))
  if [[ ${count} -eq ${retries} ]]; then
@@ -255,4 +269,4 @@ echo '✔ A/B testing promotion test passed'
 
 kubectl -n ingress-nginx logs deployment/flagger
 
-echo '✔ All tests passed'
+echo '✔ All tests passed'
diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh
@@ -3,7 +3,7 @@
 set -o errexit
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
-NGINX_HELM_VERSION=1.34.2 # ingress v0.30.0
+NGINX_HELM_VERSION=1.34.3 # ingress v0.30.0
 
 echo '>>> Installing NGINX Ingress'
 kubectl create ns ingress-nginx