Add CircleCI workflow for end-to-end testing with Kubernetes Kind

.circleci/config.yml

@@ -0,0 +1,16 @@
+version: 2.1
+jobs:
+ e2e-testing:
+ machine: true
+ steps:
+ - checkout
+ - run: test/e2e-kind.sh
+ - run: test/e2e-istio.sh
+ - run: test/e2e-build.sh
+ - run: test/e2e-tests.sh
+
+workflows:
+ version: 2
+ build-and-test:
+ jobs:
+ - e2e-testing

.travis.yml

@@ -12,12 +12,17 @@ addons:
  packages:
  - docker-ce
 
+#before_script:
+# - go get -u sigs.k8s.io/kind
+# - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+# - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+
 script:
-- set -e
-- make test-fmt
-- make test-codegen
-- go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/controller/
-- make build
+ - set -e
+ - make test-fmt
+ - make test-codegen
+ - go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/controller/
+ - make build
 
 after_success:
 - if [ -z "$DOCKER_USER" ]; then

artifacts/flagger/deployment.yaml

@@ -23,7 +23,7 @@ spec:
  containers:
  - name: flagger
  image: quay.io/stefanprodan/flagger:0.5.1
- imagePullPolicy: Always
+ imagePullPolicy: IfNotPresent
  ports:
  - name: http
  containerPort: 8080

docs/gitbook/usage/progressive-delivery.md

@@ -113,7 +113,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.0
+podinfod=quay.io/stefanprodan/podinfo:1.4.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

test/Dockerfile.kind

@@ -0,0 +1,4 @@
+FROM golang:1.11
+
+RUN go get -u sigs.k8s.io/kind
+

test/README.md

@@ -0,0 +1,24 @@
+# Flagger end-to-end testing
+
+The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind).
+
+CircleCI e2e workflow:
+
+* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh)
+* build Kubernetes Kind from master [e2e-kind.sh](e2e-kind.sh)
+* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh)
+* install latest stable Helm CLI [e2e-istio.sh](e2e-istio.sh)
+* deploy Tiller on the local cluster [e2e-istio.sh](e2e-istio.sh)
+* install Istio CRDs with Helm [e2e-istio.sh](e2e-istio.sh)
+* install Istio control plane and Prometheus with Helm [e2e-istio.sh](e2e-istio.sh)
+* build Flagger container image [e2e-build.sh](e2e-build.sh)
+* load Flagger image onto the local cluster [e2e-build.sh](e2e-build.sh)
+* deploy Flagger in the istio-system namespace [e2e-build.sh](e2e-build.sh)
+* create a test namespace with Istio injection enabled [e2e-tests.sh](e2e-tests.sh)
+* deploy the load tester in the test namespace [e2e-tests.sh](e2e-tests.sh)
+* deploy a demo workload (podinfo) in the test namespace [e2e-tests.sh](e2e-tests.sh)
+* test the canary initialization [e2e-tests.sh](e2e-tests.sh)
+* test the canary analysis and promotion [e2e-tests.sh](e2e-tests.sh)
+
+
+

test/e2e-build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Building Flagger'
+cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile
+
+echo '>>> Installing Flagger'
+kind load docker-image test/flagger:latest
+kubectl apply -f ${REPO_ROOT}/artifacts/flagger/
+kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest
+kubectl -n istio-system rollout status deployment/flagger

test/e2e-istio-values.yaml

@@ -0,0 +1,62 @@
+#
+# Minimal Istio Configuration required by Flagger
+#
+
+# pilot configuration
+pilot:
+ enabled: true
+ sidecar: true
+
+gateways:
+ enabled: false
+ istio-ingressgateway:
+ autoscaleMax: 1
+
+# citadel configuration
+security:
+ enabled: true
+
+# sidecar-injector webhook configuration
+sidecarInjectorWebhook:
+ enabled: true
+
+# galley configuration
+galley:
+ enabled: false
+
+# mixer configuration
+mixer:
+ policy:
+ enabled: false
+ replicaCount: 1
+ autoscaleEnabled: false
+ telemetry:
+ enabled: true
+ replicaCount: 1
+ autoscaleEnabled: false
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+
+# addon prometheus configuration
+prometheus:
+ enabled: true
+ scrapeInterval: 5s
+
+# addon jaeger tracing configuration
+tracing:
+ enabled: false
+
+# Common settings.
+global:
+ proxy:
+ # Resources for the sidecar.
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ limits:
+ cpu: 2000m
+ memory: 128Mi
+ useMCP: false

test/e2e-istio.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+ISTIO_VER="1.1.0-rc.0"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo ">>> Installing Helm"
+curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+
+echo '>>> Installing Tiller'
+kubectl --namespace kube-system create sa tiller
+kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+helm init --service-account tiller --upgrade --wait
+
+echo ">>> Installing Istio ${ISTIO_VER}"
+helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts
+
+echo '>>> Installing Istio CRDs'
+helm upgrade -i istio-init istio.io/istio-init --wait --namespace istio-system
+
+echo '>>> Waiting for Istio CRDs to be ready'
+kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10
+kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11
+
+echo '>>> Installing Istio control plane'
+helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml

test/e2e-kind.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo ">>> Installing kubectl"
+curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
+chmod +x kubectl && \
+sudo mv kubectl /usr/local/bin/
+
+echo ">>> Building sigs.k8s.io/kind"
+docker build -t kind:src . -f ${REPO_ROOT}/test/Dockerfile.kind
+docker create -ti --name dummy kind:src sh
+docker cp dummy:/go/bin/kind ./kind
+docker rm -f dummy
+
+echo ">>> Installing kind"
+chmod +x kind
+sudo mv kind /usr/local/bin/
+kind create cluster --wait 5m
+
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+kubectl get pods --all-namespaces
+

test/e2e-tests.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Canary initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind, Helm and Istio
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+kubectl label namespace test istio-injection=enabled
+
+echo '>>> Installing the load tester'
+kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Initialising canary'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+ name: podinfo
+ namespace: test
+spec:
+ targetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: podinfo
+ progressDeadlineSeconds: 60
+ service:
+ port: 9898
+ canaryAnalysis:
+ interval: 15s
+ threshold: 15
+ maxWeight: 50
+ stepWeight: 10
+ metrics:
+ - name: istio_requests_total
+ threshold: 99
+ interval: 1m
+ - name: istio_request_duration_seconds_bucket
+ threshold: 500
+ interval: 30s
+ webhooks:
+ - name: load-test
+ url: http://flagger-loadtester.test/
+ timeout: 5s
+ metadata:
+ cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test:9898/"
+
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+ kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+ sleep 5
+ count=$(($count + 1))
+ if [[ ${count} -eq ${retries} ]]; then
+ kubectl -n istio-system logs deployment/flagger
+ echo "No more retries left"
+ exit 1
+ fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+ kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false
+ sleep 10
+ kubectl -n istio-system logs deployment/flagger --tail 1
+ count=$(($count + 1))
+ if [[ ${count} -eq ${retries} ]]; then
+ kubectl -n test describe deployment/podinfo
+ kubectl -n test describe deployment/podinfo-primary
+ kubectl -n istio-system logs deployment/flagger
+ echo "No more retries left"
+ exit 1
+ fi
+done
+
+echo '✔ Canary promotion test passed'
+
+kubectl -n istio-system logs deployment/flagger
+
+echo '✔ All tests passed'

test/e2e-workload.yaml

@@ -0,0 +1,67 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: podinfo
+ namespace: test
+ labels:
+ app: podinfo
+spec:
+ minReadySeconds: 5
+ revisionHistoryLimit: 5
+ progressDeadlineSeconds: 60
+ strategy:
+ rollingUpdate:
+ maxUnavailable: 0
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: podinfo
+ template:
+ metadata:
+ annotations:
+ prometheus.io/scrape: "true"
+ labels:
+ app: podinfo
+ spec:
+ containers:
+ - name: podinfod
+ image: quay.io/stefanprodan/podinfo:1.4.0
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 9898
+ name: http
+ protocol: TCP
+ command:
+ - ./podinfo
+ - --port=9898
+ - --level=info
+ - --random-delay=false
+ - --random-error=false
+ env:
+ - name: PODINFO_UI_COLOR
+ value: blue
+ livenessProbe:
+ exec:
+ command:
+ - podcli
+ - check
+ - http
+ - localhost:9898/healthz
+ initialDelaySeconds: 5
+ timeoutSeconds: 5
+ readinessProbe:
+ exec:
+ command:
+ - podcli
+ - check
+ - http
+ - localhost:9898/readyz
+ initialDelaySeconds: 5
+ timeoutSeconds: 5
+ resources:
+ limits:
+ cpu: 1000m
+ memory: 128Mi
+ requests:
+ cpu: 1m
+ memory: 16Mi