-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit 4f0045a
Showing
13 changed files
with
280 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
*.tgz |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
#!/bin/bash | ||
|
||
#************************************************************ | ||
# This script starts a 4 pod deployment of BLUE servers | ||
#************************************************************ | ||
|
||
# TODO - Populate this value from the *actual* replica count | ||
REPLICA_COUNT=4 | ||
# Release color will be BLUE | ||
RELEASE_COLOR="#0000FF" | ||
|
||
helm upgrade --install tweet HttpCanary-0.1.0.tgz \ | ||
--set deployment[0].track=release \ | ||
--set deployment[0].color=$RELEASE_COLOR \ | ||
--set deployment[0].replicas=$REPLICA_COUNT \ | ||
--debug |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
#!/bin/bash | ||
|
||
#************************************************************ | ||
# This script adds an additional single pod canary deployment | ||
#************************************************************ | ||
|
||
# TODO - Populate this value from the *actual* replica count | ||
REPLICA_COUNT=4 | ||
# TODO - Populate this value from the *actual* color of release | ||
RELEASE_COLOR="#0000FF" | ||
|
||
CANARY_COLOR="#FF0000" | ||
|
||
helm upgrade --install tweet HttpCanary-0.1.0.tgz \ | ||
--set deployment[0].track=release \ | ||
--set deployment[0].color=$RELEASE_COLOR \ | ||
--set deployment[0].replicas=$REPLICA_COUNT \ | ||
--set deployment[1].track=canary \ | ||
--set deployment[1].color=$CANARY_COLOR \ | ||
--set deployment[1].replicas=1 \ | ||
--debug |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
#!/bin/bash | ||
|
||
#************************************************************ | ||
# This script simply runs a 4 pod deployment of RED servers. | ||
# This should be more complex as described below. If | ||
# implemented as is it will probably tear down the old pods | ||
# before the new pods are up and running. | ||
#************************************************************ | ||
|
||
# TODO - Populate this value from the *actual* replica count | ||
REPLICA_COUNT=4 | ||
# New release color will be RED | ||
RELEASE_COLOR="#FF0000" | ||
|
||
# This should really do the following rather than just a hard cut over | ||
# scale canary to 4 | ||
# scale release to 0 | ||
# label release as to_be_removed | ||
# label canary as release | ||
# remove to_be_removed | ||
|
||
helm upgrade --install tweet HttpCanary-0.1.0.tgz \ | ||
--set deployment[0].track=release \ | ||
--set deployment[0].color=$RELEASE_COLOR \ | ||
--set deployment[0].replicas=$REPLICA_COUNT \ | ||
--debug |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
#!/bin/bash | ||
|
||
#************************************************************ | ||
# This script adds a 1 pod canary deployment of BLUE servers | ||
#************************************************************ | ||
|
||
# TODO - Populate this value from the *actual* replica count | ||
REPLICA_COUNT=4 | ||
# TODO - Populate this value from the *actual* color of release | ||
RELEASE_COLOR="#FF0000" | ||
|
||
CANARY_COLOR="#0000FF" | ||
|
||
helm upgrade --install tweet HttpCanary-0.1.0.tgz \ | ||
--set deployment[0].track=release \ | ||
--set deployment[0].color=$RELEASE_COLOR \ | ||
--set deployment[0].replicas=$REPLICA_COUNT \ | ||
--set deployment[1].track=canary \ | ||
--set deployment[1].color=$CANARY_COLOR \ | ||
--set deployment[1].replicas=1 \ | ||
--debug |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
apiVersion: v1 | ||
description: An http server to demonstrate canary deployments | ||
name: HttpCanary | ||
version: 0.1.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,120 @@ | ||
# helm-canary | ||
|
||
Sample project to demonstrate how to achieve canary deployments of helm charts. This uses simple load balancing of pods using a NodePort based on pod numbers. | ||
|
||
The principle is to run multiple deployments. One for the release version and one for the canary version. Once the canary is validated as stable pods are scaled and the old release version pods removed. | ||
|
||
This has been verified with Helm v2.11.0, Minikube v0.30.0 and Kubectl v1.11 | ||
|
||
# Prerequisites | ||
|
||
Minikube up and running, Helm and Tiller installed | ||
|
||
# Usage | ||
|
||
## 1. Start the cluster | ||
|
||
This will start a service and 4 "blue" pods. This will also output the configuration that will be used. | ||
|
||
`> ./1_release_to_blue.sh` | ||
|
||
## 2. Find out the service URL | ||
|
||
``` | ||
➜ helm-canary git:(master) ✗ ./status.sh | ||
NAME READY STATUS RESTARTS AGE IP NODE | ||
http-canary-deployment-canary-6c9f8dc59-vgdxm 1/1 Running 0 5h 172.17.0.13 minikube | ||
http-canary-deployment-release-6fcf77d6f7-b46jr 1/1 Running 0 5h 172.17.0.15 minikube | ||
http-canary-deployment-release-6fcf77d6f7-ghm5z 1/1 Running 0 5h 172.17.0.17 minikube | ||
http-canary-deployment-release-6fcf77d6f7-p579s 1/1 Running 0 5h 172.17.0.16 minikube | ||
http-canary-deployment-release-6fcf77d6f7-pj6kj 1/1 Running 0 5h 172.17.0.14 minikube | ||
Service available at http://192.168.99.100:31708 | ||
``` | ||
|
||
Once you have a URL you can test the reponse in a browser or using curl | ||
``` | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31708 | ||
<html onclick="window.location.href = '/die'" style='background:#0000FF'> Requested: / | ||
</html>% | ||
``` | ||
|
||
You can always find out the state of the pods with `kc get all -l category=microservices` or just the pods `kc get pods -l category=microservices` | ||
|
||
## 3. Canary deploy red servers | ||
|
||
`> ./2_canary_to_red.sh` | ||
|
||
This will create a single canary pod and leave the release pods unchanged. You can watch until the new pod is running using `kc get pods -l category=microservices` | ||
|
||
Multiple curl requests or browser refreshes will show the canary being called | ||
|
||
``` | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#0000FF'> Requested: / | ||
</html>% | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#0000FF'> Requested: / | ||
</html>% | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#FF0000'> Requested: / | ||
</html>% | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#0000FF'> Requested: / | ||
</html>% | ||
``` | ||
|
||
## 4. Full red release | ||
|
||
`> ./3_release_to_red.sh ` | ||
|
||
You can now watch until all pods are updated with red release pods. | ||
|
||
``` | ||
➜ helm-canary git:(master) ✗ kc get pods -l category=microservices | ||
NAME READY STATUS RESTARTS AGE | ||
http-canary-deployment-release-6fcf77d6f7-6fl49 1/1 Running 0 30s | ||
http-canary-deployment-release-6fcf77d6f7-6tp5j 1/1 Running 0 35s | ||
http-canary-deployment-release-6fcf77d6f7-7sp9b 1/1 Running 0 48s | ||
http-canary-deployment-release-6fcf77d6f7-f4ph5 1/1 Running 0 49s | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#FF0000'> Requested: / | ||
</html>% | ||
``` | ||
|
||
## 5. Canary back to blue | ||
|
||
`> 4_canary_to_blue.sh` | ||
|
||
A single blue canary will now be created | ||
|
||
``` | ||
➜ helm-canary git:(master) ✗ kc get pods -l category=microservices | ||
NAME READY STATUS RESTARTS AGE | ||
http-canary-deployment-canary-6c9f8dc59-q9bdv 1/1 Running 0 11s | ||
http-canary-deployment-release-6fcf77d6f7-6fl49 1/1 Running 0 1m | ||
http-canary-deployment-release-6fcf77d6f7-6tp5j 1/1 Running 0 1m | ||
http-canary-deployment-release-6fcf77d6f7-7sp9b 1/1 Running 0 2m | ||
http-canary-deployment-release-6fcf77d6f7-f4ph5 1/1 Running 0 2m | ||
``` | ||
|
||
You can see it being called | ||
``` | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#FF0000'> Requested: / | ||
</html>% | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#FF0000'> Requested: / | ||
</html>% | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#FF0000'> Requested: / | ||
</html>% | ||
➜ helm-canary git:(master) ✗ curl http://192.168.99.100:31167 | ||
<html onclick="window.location.href = '/die'" style='background:#0000FF'> Requested: / | ||
</html>% | ||
``` | ||
|
||
## 6. Full blue release | ||
|
||
You can now follow step 1. and return to a full blue release. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
#!/bin/bash | ||
|
||
helm delete tweet --purge |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
#!/bin/bash | ||
helm package . |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
#!/bin/bash | ||
|
||
#************************************************************ | ||
# This script outputs the url of the service | ||
#************************************************************ | ||
|
||
kubectl get pods -o wide | ||
|
||
SERVICE_PORT=$(kubectl get svc http-canary-service -o go-template='{{range .spec.ports}}{{if .nodePort}}{{.nodePort}}{{"\n"}}{{end}}{{end}}') | ||
IP=$(minikube ip) | ||
|
||
echo "Service available at http://$IP:$SERVICE_PORT" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
{{ $local := . }} | ||
{{- range $index, $track := .Values.deployment }} | ||
apiVersion: apps/v1 | ||
kind: Deployment | ||
metadata: | ||
name: http-canary-deployment-{{$track.track}} | ||
labels: | ||
app: http-canary | ||
color: {{$track.track}} | ||
category: microservices | ||
spec: | ||
replicas: {{$track.replicas}} | ||
selector: | ||
matchLabels: | ||
app: http-canary | ||
color: {{$track.track}} | ||
template: | ||
metadata: | ||
labels: | ||
app: http-canary | ||
color: {{$track.track}} | ||
category: microservices | ||
spec: | ||
containers: | ||
- name: http-canary-{{$track.track}} | ||
image: paulhopkins/http-canary:latest | ||
ports: | ||
- containerPort: 80 | ||
command: | ||
- "./myapp" | ||
- "{{$track.color}}" | ||
--- | ||
{{ end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
apiVersion: v1 | ||
kind: Service | ||
metadata: | ||
name: http-canary-service | ||
labels: | ||
category: microservices | ||
spec: | ||
ports: | ||
- port: 80 | ||
targetPort: 80 | ||
protocol: TCP | ||
# the set of pods to load balance traffic to. | ||
selector: | ||
app: http-canary | ||
type: NodePort |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
# Default values include a 4 pod release deployment | ||
|
||
deployment: | ||
- track: release | ||
color: "#0000FF" | ||
replicas: 4 |