From 298d8c2d650a94411e82ec3c5eb1a4a4ba6e2aee Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 5 Oct 2019 11:21:43 +0300 Subject: [PATCH 1/3] Allow gPRC protocol for App Mesh Use the canary service port name to set http or grpc protocol on App Mesh virtual nodes and virtual routers --- artifacts/appmesh/canary.yaml | 3 + .../usage/appmesh-progressive-delivery.md | 3 + pkg/router/appmesh.go | 66 +++++++++++-------- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/artifacts/appmesh/canary.yaml b/artifacts/appmesh/canary.yaml index 0cea6d6a4..05fec1611 100644 --- a/artifacts/appmesh/canary.yaml +++ b/artifacts/appmesh/canary.yaml @@ -20,6 +20,9 @@ spec: service: # container port port: 9898 + # container port name (optional) + # can be http or grpc + portName: http # App Mesh reference meshName: global # define the canary analysis timing and KPIs diff --git a/docs/gitbook/usage/appmesh-progressive-delivery.md b/docs/gitbook/usage/appmesh-progressive-delivery.md index 66b076e29..6b13f2882 100644 --- a/docs/gitbook/usage/appmesh-progressive-delivery.md +++ b/docs/gitbook/usage/appmesh-progressive-delivery.md @@ -67,6 +67,9 @@ spec: service: # container port port: 9898 + # container port name (optional) + # can be http or grpc + portName: http # App Mesh reference meshName: global # App Mesh egress (optional) diff --git a/pkg/router/appmesh.go b/pkg/router/appmesh.go index 57b0f5054..f57719961 100644 --- a/pkg/router/appmesh.go +++ b/pkg/router/appmesh.go @@ -2,16 +2,19 @@ package router import ( "fmt" + "strings" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - AppmeshV1beta1 "github.com/weaveworks/flagger/pkg/apis/appmesh/v1beta1" - flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" - clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned" "go.uber.org/zap" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" + + appmeshv1 "github.com/weaveworks/flagger/pkg/apis/appmesh/v1beta1" + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned" ) // AppMeshRouter is managing AppMesh virtual services @@ -76,27 +79,28 @@ func (ar *AppMeshRouter) Reconcile(canary *flaggerv1.Canary) error { // reconcileVirtualNode creates or updates a virtual node // the virtual node naming format is name-role-namespace func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name string, host string) error { - vnSpec := AppmeshV1beta1.VirtualNodeSpec{ + protocol := getProtocol(canary) + vnSpec := appmeshv1.VirtualNodeSpec{ MeshName: canary.Spec.Service.MeshName, - Listeners: []AppmeshV1beta1.Listener{ + Listeners: []appmeshv1.Listener{ { - PortMapping: AppmeshV1beta1.PortMapping{ + PortMapping: appmeshv1.PortMapping{ Port: int64(canary.Spec.Service.Port), - Protocol: "http", + Protocol: protocol, }, }, }, - ServiceDiscovery: &AppmeshV1beta1.ServiceDiscovery{ - Dns: &AppmeshV1beta1.DnsServiceDiscovery{ + ServiceDiscovery: &appmeshv1.ServiceDiscovery{ + Dns: &appmeshv1.DnsServiceDiscovery{ HostName: host, }, }, } - backends := []AppmeshV1beta1.Backend{} + backends := []appmeshv1.Backend{} for _, b := range canary.Spec.Service.Backends { - backend := AppmeshV1beta1.Backend{ - VirtualService: AppmeshV1beta1.VirtualServiceBackend{ + backend := appmeshv1.Backend{ + VirtualService: appmeshv1.VirtualServiceBackend{ VirtualServiceName: b, }, } @@ -110,7 +114,7 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str // create virtual node if errors.IsNotFound(err) { - virtualnode = &AppmeshV1beta1.VirtualNode{ + virtualnode = &appmeshv1.VirtualNode{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: canary.Namespace, @@ -159,6 +163,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name targetName := canary.Spec.TargetRef.Name canaryVirtualNode := fmt.Sprintf("%s-canary", targetName) primaryVirtualNode := fmt.Sprintf("%s-primary", targetName) + protocol := getProtocol(canary) // App Mesh supports only URI prefix routePrefix := "/" @@ -168,28 +173,28 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name routePrefix = canary.Spec.Service.Match[0].Uri.Prefix } - vsSpec := AppmeshV1beta1.VirtualServiceSpec{ + vsSpec := appmeshv1.VirtualServiceSpec{ MeshName: canary.Spec.Service.MeshName, - VirtualRouter: &AppmeshV1beta1.VirtualRouter{ + VirtualRouter: &appmeshv1.VirtualRouter{ Name: fmt.Sprintf("%s-router", targetName), - Listeners: []AppmeshV1beta1.Listener{ + Listeners: []appmeshv1.Listener{ { - PortMapping: AppmeshV1beta1.PortMapping{ + PortMapping: appmeshv1.PortMapping{ Port: int64(canary.Spec.Service.Port), - Protocol: "http", + Protocol: protocol, }, }, }, }, - Routes: []AppmeshV1beta1.Route{ + Routes: []appmeshv1.Route{ { Name: fmt.Sprintf("%s-route", targetName), - Http: &AppmeshV1beta1.HttpRoute{ - Match: AppmeshV1beta1.HttpRouteMatch{ + Http: &appmeshv1.HttpRoute{ + Match: appmeshv1.HttpRouteMatch{ Prefix: routePrefix, }, - Action: AppmeshV1beta1.HttpRouteAction{ - WeightedTargets: []AppmeshV1beta1.WeightedTarget{ + Action: appmeshv1.HttpRouteAction{ + WeightedTargets: []appmeshv1.WeightedTarget{ { VirtualNodeName: canaryVirtualNode, Weight: canaryWeight, @@ -209,7 +214,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name // create virtual service if errors.IsNotFound(err) { - virtualService = &AppmeshV1beta1.VirtualService{ + virtualService = &appmeshv1.VirtualService{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: canary.Namespace, @@ -238,7 +243,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name // update virtual service but keep the original target weights if virtualService != nil { - if diff := cmp.Diff(vsSpec, virtualService.Spec, cmpopts.IgnoreTypes(AppmeshV1beta1.WeightedTarget{})); diff != "" { + if diff := cmp.Diff(vsSpec, virtualService.Spec, cmpopts.IgnoreTypes(appmeshv1.WeightedTarget{})); diff != "" { vsClone := virtualService.DeepCopy() vsClone.Spec = vsSpec vsClone.Spec.Routes[0].Http.Action = virtualService.Spec.Routes[0].Http.Action @@ -317,8 +322,8 @@ func (ar *AppMeshRouter) SetRoutes( } vsClone := vs.DeepCopy() - vsClone.Spec.Routes[0].Http.Action = AppmeshV1beta1.HttpRouteAction{ - WeightedTargets: []AppmeshV1beta1.WeightedTarget{ + vsClone.Spec.Routes[0].Http.Action = appmeshv1.HttpRouteAction{ + WeightedTargets: []appmeshv1.WeightedTarget{ { VirtualNodeName: fmt.Sprintf("%s-canary", targetName), Weight: int64(canaryWeight), @@ -337,3 +342,10 @@ func (ar *AppMeshRouter) SetRoutes( return nil } + +func getProtocol(canary *flaggerv1.Canary) string { + if strings.Contains(canary.Spec.Service.PortName, "grpc") { + return "grpc" + } + return "http" +} From 3103bde7f7db9c28b1e5713eb90c96c7181b5082 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 5 Oct 2019 11:52:41 +0300 Subject: [PATCH 2/3] Use the App Mesh Prometheus chart in docs --- .../install/flagger-install-on-eks-appmesh.md | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/gitbook/install/flagger-install-on-eks-appmesh.md b/docs/gitbook/install/flagger-install-on-eks-appmesh.md index c2326a99a..a23f90fc6 100644 --- a/docs/gitbook/install/flagger-install-on-eks-appmesh.md +++ b/docs/gitbook/install/flagger-install-on-eks-appmesh.md @@ -17,8 +17,7 @@ The App Mesh integration with EKS is made out of the following components: ### Create a Kubernetes cluster In order to create an EKS cluster you can use [eksctl](https://eksctl.io). -Eksctl is an open source command-line utility made by Weaveworks in collaboration with Amazon, -it’s a Kubernetes-native tool written in Go. +Eksctl is an open source command-line utility made by Weaveworks in collaboration with Amazon. On MacOS you can install eksctl with Homebrew: @@ -137,7 +136,17 @@ Status: Type: MeshActive ``` -### Install Flagger, Prometheus and Grafana +In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, +you'll need to setup a Prometheus instance to scrape the Envoy sidecars. + +Install the App Mesh Prometheus: + +```sh +helm upgrade -i appmesh-prometheus eks/appmesh-prometheus \ +--wait --namespace appmesh-system +``` + +### Install Flagger and Grafana Add Flagger Helm repository: @@ -151,20 +160,17 @@ Install Flagger's Canary CRD: kubectl apply -f https://raw.githubusercontent.com/weaveworks/flagger/master/artifacts/flagger/crd.yaml ``` -Deploy Flagger and Prometheus in the _**appmesh-system**_ namespace: +Deploy Flagger in the _**appmesh-system**_ namespace: ```bash helm upgrade -i flagger flagger/flagger \ --namespace=appmesh-system \ --set crd.create=false \ --set meshProvider=appmesh \ ---set prometheus.install=true +--set metricsServer=appmesh-prometheus:9090 ``` -In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, -you'll need to setup a Prometheus instance to scrape the Envoy sidecars. - -You can enable **Slack** notifications with: +You can enable Slack or MS Teams notifications with: ```bash helm upgrade -i flagger flagger/flagger \ @@ -181,7 +187,7 @@ Deploy Grafana in the _**appmesh-system**_ namespace: ```bash helm upgrade -i flagger-grafana flagger/grafana \ --namespace=appmesh-system \ ---set url=http://flagger-prometheus.appmesh-system:9090 +--set url=http://appmesh-prometheus:9090 ``` You can access Grafana using port forwarding: From 76800d0ed0461105f7ca2a8782ebb305ae4974f9 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 5 Oct 2019 12:15:54 +0300 Subject: [PATCH 3/3] Update canary spec in docs --- README.md | 7 +++-- docs/gitbook/how-it-works.md | 51 ++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index aaa3c60e5..637c5737d 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,6 @@ metadata: spec: # service mesh provider (optional) # can be: kubernetes, istio, linkerd, appmesh, nginx, gloo, supergloo - # use the kubernetes provider for Blue/Green style deployments provider: istio # deployment reference targetRef: @@ -94,6 +93,10 @@ spec: # Istio virtual service host names (optional) hosts: - podinfo.example.com + # Istio traffic policy (optional) + trafficPolicy: + tls: + mode: ISTIO_MUTUAL # HTTP match conditions (optional) match: - uri: @@ -144,7 +147,7 @@ spec: topic="podinfo" }[1m] ) - # external checks (optional) + # testing (optional) webhooks: - name: load-test url: http://flagger-loadtester.test/ diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 53f7156b5..6669cf4ad 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -19,7 +19,6 @@ metadata: spec: # service mesh provider (optional) # can be: kubernetes, istio, linkerd, appmesh, nginx, gloo, supergloo - # use the kubernetes provider for Blue/Green style deployments provider: istio # deployment reference targetRef: @@ -38,13 +37,7 @@ spec: # container port port: 9898 # service port name (optional, will default to "http") - portName: http-podinfo - # Istio gateways (optional) - gateways: - - public-gateway.istio-system.svc.cluster.local - # Istio virtual service host names (optional) - hosts: - - podinfo.example.com + portName: http # promote the canary without analysing it (default false) skipAnalysis: false # define the canary analysis timing and KPIs @@ -71,15 +64,13 @@ spec: # milliseconds threshold: 500 interval: 30s - # external checks (optional) + # testing (optional) webhooks: - - name: integration-tests - url: http://podinfo.test:9898/echo - timeout: 1m - # key-value pairs (optional) + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s metadata: - test: "all" - token: "16688eb5e9f289f1991c" + cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/" ``` **Note** that the target deployment must have a single label selector in the format `app: `: @@ -102,8 +93,8 @@ spec: Besides `app` Flagger supports `name` and `app.kubernetes.io/name` selectors. If you use a different convention you can specify your label with the `-selector-labels` flag. -The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Service and -the Istio Virtual Service. The container port from the target deployment should match the `service.port` value. +The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Services. +The container port from the target deployment should match the `service.port` value. ### Canary status @@ -201,10 +192,11 @@ spec: # Istio virtual service host names (optional) hosts: - frontend.example.com - # Istio traffic policy (optional) + # Istio traffic policy trafficPolicy: - loadBalancer: - simple: LEAST_CONN + tls: + # use ISTIO_MUTUAL when mTLS is enabled + mode: DISABLE # HTTP match conditions (optional) match: - uri: @@ -291,8 +283,8 @@ metadata: spec: host: frontend-primary trafficPolicy: - loadBalancer: - simple: LEAST_CONN + tls: + mode: DISABLE --- apiVersion: networking.istio.io/v1alpha3 kind: DestinationRule @@ -302,15 +294,15 @@ metadata: spec: host: frontend-canary trafficPolicy: - loadBalancer: - simple: LEAST_CONN + tls: + mode: DISABLE ``` Flagger keeps in sync the virtual service and destination rules with the canary service spec. Any direct modification to the virtual service spec will be overwritten. To expose a workload inside the mesh on `http://backend.test.svc.cluster.local:9898`, -the service spec can contain only the container port: +the service spec can contain only the container port and the traffic policy: ```yaml apiVersion: flagger.app/v1alpha3 @@ -321,6 +313,9 @@ metadata: spec: service: port: 9898 + trafficPolicy: + tls: + mode: DISABLE ``` Based on the above spec, Flagger will create several ClusterIP services like: @@ -531,7 +526,7 @@ sum( ) ``` -App Mesh query: +Envoy query (App Mesh or Gloo): ```javascript sum( @@ -539,7 +534,7 @@ sum( envoy_cluster_upstream_rq{ kubernetes_namespace="$namespace", kubernetes_pod_name=~"$workload", - response_code!~"5.*" + envoy_response_code!~"5.*" }[$interval] ) ) @@ -584,7 +579,7 @@ histogram_quantile(0.99, ) ``` -App Mesh query: +Envoy query (App Mesh or Gloo): ```javascript histogram_quantile(0.99,