Implement helm and bash pre-rollout hooks

Dockerfile.loadtester

@@ -21,6 +21,10 @@ WORKDIR /home/app
 RUN curl -sSLo hey "https://storage.googleapis.com/jblabs/dist/hey_linux_v0.1.2" && \
 chmod +x hey && mv hey /usr/local/bin/hey
 
+RUN curl -sSL "https://get.helm.sh/helm-v2.12.3-linux-amd64.tar.gz" | tar xvz && \
+chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helm && \
+rm -rf linux-amd64
+
 COPY --from=builder /go/src/github.com/weaveworks/flagger/loadtester .
 
 RUN chown -R app:app ./

Makefile

@@ -7,7 +7,7 @@ LT_VERSION?=$(shell grep 'VERSION' cmd/loadtester/main.go | awk '{ print $$4 }'
 TS=$(shell date +%Y-%m-%d_%H-%M-%S)
 
 run:
- go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info \
+ go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
  -metrics-server=https://prometheus.istio.weavedx.com \
  -slack-url=https://hooks.slack.com/services/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 \
  -slack-channel="devops-alerts"
@@ -106,6 +106,11 @@ reset-test:
  kubectl apply -f ./artifacts/namespaces
  kubectl apply -f ./artifacts/canaries
 
+loadtester-run:
+ docker build -t weaveworks/flagger-loadtester:$(LT_VERSION) . -f Dockerfile.loadtester
+ docker rm -f tester || true
+ docker run -dp 8888:9090 --name tester weaveworks/flagger-loadtester:$(LT_VERSION)
+
 loadtester-push:
  docker build -t weaveworks/flagger-loadtester:$(LT_VERSION) . -f Dockerfile.loadtester
- docker push weaveworks/flagger-loadtester:$(LT_VERSION)
+ docker push weaveworks/flagger-loadtester:$(LT_VERSION)

artifacts/canaries/canary.yaml

@@ -73,5 +73,5 @@ spec:
  timeout: 5s
  metadata:
  type: cmd
- cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
+ cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/"
  logCmdOutput: "true"

artifacts/helmtester/deployment.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flagger-helmtester
+ namespace: kube-system
+ labels:
+ app: flagger-helmtester
+spec:
+ selector:
+ matchLabels:
+ app: flagger-helmtester
+ template:
+ metadata:
+ labels:
+ app: flagger-helmtester
+ annotations:
+ prometheus.io/scrape: "true"
+ spec:
+ serviceAccountName: tiller
+ containers:
+ - name: helmtester
+ image: weaveworks/flagger-loadtester:0.4.0
+ imagePullPolicy: IfNotPresent
+ ports:
+ - name: http
+ containerPort: 8080
+ command:
+ - ./loadtester
+ - -port=8080
+ - -log-level=info
+ - -timeout=1h
+ livenessProbe:
+ exec:
+ command:
+ - wget
+ - --quiet
+ - --tries=1
+ - --timeout=4
+ - --spider
+ - http://localhost:8080/healthz
+ timeoutSeconds: 5
+ readinessProbe:
+ exec:
+ command:
+ - wget
+ - --quiet
+ - --tries=1
+ - --timeout=4
+ - --spider
+ - http://localhost:8080/healthz
+ timeoutSeconds: 5
+ resources:
+ limits:
+ memory: "512Mi"
+ cpu: "1000m"
+ requests:
+ memory: "32Mi"
+ cpu: "10m"

artifacts/helmtester/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flagger-helmtester
+ namespace: kube-system
+ labels:
+ app: flagger-helmtester
+spec:
+ type: ClusterIP
+ selector:
+ app: flagger-helmtester
+ ports:
+ - name: http
+ port: 80
+ protocol: TCP
+ targetPort: http

charts/loadtester/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: loadtester
-version: 0.4.0
-appVersion: 0.3.0
+version: 0.4.1
+appVersion: 0.4.0
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger's load testing services based on rakyll/hey that generates traffic during canary analysis when configured as a webhook.

charts/loadtester/README.md

@@ -7,7 +7,6 @@ and can be used to generates traffic during canary analysis when configured as a
 ## Prerequisites
 
 * Kubernetes >= 1.11
-* Istio >= 1.0
 
 ## Installing the Chart
 
@@ -25,7 +24,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester
 
 The command deploys Grafana on the Kubernetes cluster in the default namespace.
 
-> **Tip**: Note that the namespace where you deploy the load tester should have the Istio sidecar injection enabled
+> **Tip**: Note that the namespace where you deploy the load tester should have the Istio or App Mesh sidecar injection enabled
 
 The [configuration](#configuration) section lists the parameters that can be configured during installation.
 
@@ -48,13 +47,14 @@ Parameter | Description | Default
 `image.repository` | Image repository | `quay.io/stefanprodan/flagger-loadtester`
 `image.pullPolicy` | Image pull policy | `IfNotPresent`
 `image.tag` | Image tag | `<VERSION>`
-`replicaCount` | desired number of pods | `1`
+`replicaCount` | Desired number of pods | `1`
+`serviceAccountName` | Kubernetes service account name | `none` 
 `resources.requests.cpu` | CPU requests | `10m`
-`resources.requests.memory` | memory requests | `64Mi`
+`resources.requests.memory` | Memory requests | `64Mi`
 `tolerations` | List of node taints to tolerate | `[]`
 `affinity` | node/pod affinities | `node`
-`nodeSelector` | node labels for pod assignment | `{}`
-`service.type` | type of service | `ClusterIP`
+`nodeSelector` | Node labels for pod assignment | `{}`
+`service.type` | Type of service | `ClusterIP`
 `service.port` | ClusterIP port | `80`
 `cmd.timeout` | Command execution timeout | `1h`
 `logLevel` | Log level can be debug, info, warning, error or panic | `info`

charts/loadtester/templates/deployment.yaml

@@ -19,6 +19,9 @@ spec:
  annotations:
  appmesh.k8s.aws/ports: "444"
  spec:
+ {{- if .Values.serviceAccountName }}
+ serviceAccountName: {{ .Values.serviceAccountName }}
+ {{- end }}
  containers:
  - name: {{ .Chart.Name }}
  image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"

charts/loadtester/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 
 image:
  repository: weaveworks/flagger-loadtester
- tag: 0.3.0
+ tag: 0.4.0
  pullPolicy: IfNotPresent
 
 logLevel: info
@@ -27,6 +27,8 @@ tolerations: []
 
 affinity: {}
 
+serviceAccountName: ""
+
 # App Mesh virtual node settings
 meshName: ""
 #backends:

charts/podinfo/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
-version: 2.0.1
+version: 2.1.0
 appVersion: 1.4.0
 name: podinfo
 engine: gotpl

charts/podinfo/templates/canary.yaml

@@ -38,8 +38,17 @@ spec:
  - name: request-duration
  threshold: {{ .Values.canary.thresholds.latency }}
  interval: 1m
- {{- if .Values.canary.loadtest.enabled }}
  webhooks:
+ {{- if .Values.canary.helmtest.enabled }}
+ - name: "helm test"
+ type: pre-rollout
+ url: {{ .Values.canary.helmtest.url }}
+ timeout: 3m
+ metadata:
+ type: "helm"
+ cmd: "test {{ .Release.Name }} --cleanup"
+ {{- end }}
+ {{- if .Values.canary.loadtest.enabled }}
  - name: load-test-get
  url: {{ .Values.canary.loadtest.url }}
  timeout: 5s
@@ -50,5 +59,5 @@ spec:
  timeout: 5s
  metadata:
  cmd: "hey -z 1m -q 5 -c 2 -m POST -d '{\"test\": true}' http://{{ template "podinfo.fullname" . }}.{{ .Release.Namespace }}:{{ .Values.service.port }}/echo"
- {{- end }}
+  {{- end }}
 {{- end }}

charts/podinfo/values.yaml

@@ -45,6 +45,10 @@ canary:
  enabled: false
  # load tester address
  url: http://flagger-loadtester.test/
+ helmtest:
+ enabled: false
+ # helm tester address
+ url: http://flagger-helmtester.kube-system/
 
 resources:
  limits:

cmd/loadtester/main.go

@@ -10,7 +10,7 @@ import (
  "time"
 )
 
-var VERSION = "0.3.0"
+var VERSION = "0.4.0"
 var (
  logLevel string
  port string

docs/gitbook/how-it-works.md

@@ -593,11 +593,11 @@ Spec:
  webhooks:
  - name: "smoke test"
  type: pre-rollout
- url: http://migration-check.db/query
- timeout: 30s
+ url: http://flagger-helmtester.kube-system/
+ timeout: 3m
  metadata:
- key1: "val1"
- key2: "val2"
+ type: "helm"
+ cmd: "test podinfo --cleanup"
  - name: "load test"
  type: rollout
  url: http://flagger-loadtester.test/
@@ -640,7 +640,7 @@ On a non-2xx response Flagger will include the response body (if any) in the fai
 For workloads that are not receiving constant traffic Flagger can be configured with a webhook, 
 that when called, will start a load test for the target workload.
 If the target workload doesn't receive any traffic during the canary analysis, 
-Flagger metric checks will fail with "no values found for metric istio_requests_total".
+Flagger metric checks will fail with "no values found for metric request-success-rate".
 
 Flagger comes with a load testing service based on [rakyll/hey](https://github.com/rakyll/hey) 
 that generates traffic during analysis when configured as a webhook.
@@ -677,18 +677,18 @@ webhooks:
  timeout: 5s
  metadata:
  type: cmd
- cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
+ cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/"
  - name: load-test-post
  url: http://flagger-loadtester.test/
  timeout: 5s
  metadata:
  type: cmd
- cmd: "hey -z 1m -q 10 -c 2 -m POST -d '{test: 2}' http://podinfo.test:9898/echo"
+ cmd: "hey -z 1m -q 10 -c 2 -m POST -d '{test: 2}' http://podinfo-canary.test:9898/echo"
 ```
 
 When the canary analysis starts, Flagger will call the webhooks and the load tester will run the `hey` commands 
 in the background, if they are not already running. This will ensure that during the 
-analysis, the `podinfo.test` virtual service will receive a steady stream of GET and POST requests.
+analysis, the `podinfo-canary.test` service will receive a steady stream of GET and POST requests.
 
 If your workload is exposed outside the mesh with the Istio Gateway and TLS you can point `hey` to the 
 public URL and use HTTP2.
@@ -707,7 +707,7 @@ The load tester can run arbitrary commands as long as the binary is present in t
 For example if you you want to replace `hey` with another CLI, you can create your own Docker image:
 
 ```dockerfile
-FROM quay.io/stefanprodan/flagger-loadtester:<VER>
+FROM weaveworks/flagger-loadtester:<VER>
 
 RUN curl -Lo /usr/local/bin/my-cli https://github.com/user/repo/releases/download/ver/my-cli \
  && chmod +x /usr/local/bin/my-cli
@@ -741,3 +741,52 @@ webhooks:
 When the canary analysis starts, the load tester will initiate a [clone_and_start request](https://github.com/naver/ngrinder/wiki/REST-API-PerfTest)
 to the nGrinder server and start a new performance test. the load tester will periodically poll the nGrinder server
 for the status of the test, and prevent duplicate requests from being sent in subsequent analysis loops.
+
+### Integration Testing
+
+Flagger comes with a testing service that can run Helm tests or Bats tests when configured as a webhook.
+
+Deploy the Helm test runner in the `kube-system` namespace using the `tiller` service account:
+
+```bash
+helm repo add flagger https://flagger.app
+
+helm upgrade -i flagger-helmtester flagger/loadtester \
+--namespace=kube-system \
+--set serviceAccountName=tiller
+```
+
+When deployed the Helm tester API will be available at `http://flagger-helmtester.kube-system/`. 
+
+Now you can add pre-rollout webhooks to the canary analysis spec:
+
+```yaml
+ canaryAnalysis:
+ webhooks:
+ - name: "smoke test"
+ type: pre-rollout
+ url: http://flagger-helmtester.kube-system/
+ timeout: 3m
+ metadata:
+ type: "helm"
+ cmd: "test {{ .Release.Name }} --cleanup"
+```
+
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
+If the helm test fails, Flagger will retry until the analysis threshold is reached and the canary is rolled back.
+
+As an alternative to Helm you can use the [Bash Automated Testing System](https://github.com/bats-core/bats-core) to run your tests. 
+
+```yaml
+ canaryAnalysis:
+ webhooks:
+ - name: "acceptance tests"
+ type: pre-rollout
+ url: http://flagger-batstester.default/
+ timeout: 5m
+ metadata:
+ type: "bash"
+ cmd: "bats /tests/acceptance.bats"
+```
+
+Note that you should create a ConfigMap with your Bats tests and mount it inside the tester container.