From 8d1cc83405226ce257448e901855c1d1522cd814 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 19 Jun 2019 12:01:02 +0300 Subject: [PATCH 1/4] Add a no-operation router To be used for Kubernetes blue/green deployments (no service mesh or ingress controller) --- pkg/router/nop.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 pkg/router/nop.go diff --git a/pkg/router/nop.go b/pkg/router/nop.go new file mode 100644 index 000000000..66f1d8120 --- /dev/null +++ b/pkg/router/nop.go @@ -0,0 +1,24 @@ +package router + +import ( + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" +) + +// NopRouter no-operation router +type NopRouter struct { +} + +func (*NopRouter) Reconcile(canary *flaggerv1.Canary) error { + return nil +} + +func (*NopRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int) error { + return nil +} + +func (*NopRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, err error) { + if canary.Status.Iterations > 0 { + return 0, 100, nil + } + return 100, 0, nil +} From 9fada306f0a5bd67c94f3531af20ba6eeaa775d2 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 19 Jun 2019 12:02:40 +0300 Subject: [PATCH 2/4] Add a service mesh provider of type none To be used for Kubernetes blue/green deployments with the no-operations router --- Makefile | 24 +++++++++--------------- pkg/router/factory.go | 2 ++ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 51f4e097d..643be532b 100644 --- a/Makefile +++ b/Makefile @@ -8,33 +8,27 @@ TS=$(shell date +%Y-%m-%d_%H-%M-%S) run: go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \ - -metrics-server=https://prometheus.istio.weavedx.com \ - -slack-url=https://hooks.slack.com/services/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 \ - -slack-channel="devops-alerts" + -metrics-server=https://prometheus.istio.weavedx.com run-appmesh: go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=appmesh \ - -metrics-server=http://acfc235624ca911e9a94c02c4171f346-1585187926.us-west-2.elb.amazonaws.com:9090 \ - -slack-url=https://hooks.slack.com/services/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 \ - -slack-channel="devops-alerts" + -metrics-server=http://acfc235624ca911e9a94c02c4171f346-1585187926.us-west-2.elb.amazonaws.com:9090 run-nginx: go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=nginx -namespace=nginx \ - -metrics-server=http://prometheus-weave.istio.weavedx.com \ - -slack-url=https://hooks.slack.com/services/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 \ - -slack-channel="devops-alerts" + -metrics-server=http://prometheus-weave.istio.weavedx.com run-smi: go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=smi:istio -namespace=smi \ - -metrics-server=https://prometheus.istio.weavedx.com \ - -slack-url=https://hooks.slack.com/services/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 \ - -slack-channel="devops-alerts" + -metrics-server=https://prometheus.istio.weavedx.com run-gloo: go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=gloo -namespace=gloo \ - -metrics-server=https://prometheus.istio.weavedx.com \ - -slack-url=https://hooks.slack.com/services/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 \ - -slack-channel="devops-alerts" + -metrics-server=https://prometheus.istio.weavedx.com + +run-nop: + go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=none -namespace=bg \ + -metrics-server=https://prometheus.istio.weavedx.com build: docker build -t weaveworks/flagger:$(TAG) . -f Dockerfile diff --git a/pkg/router/factory.go b/pkg/router/factory.go index c330dbac1..0a13eeade 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -45,6 +45,8 @@ func (factory *Factory) KubernetesRouter(label string, ports *map[string]int32) // MeshRouter returns a service mesh router func (factory *Factory) MeshRouter(provider string) Interface { switch { + case provider == "none": + return &NopRouter{} case provider == "nginx": return &IngressRouter{ logger: factory.logger, From 7e72d23b60f8bd27f063b25af3df10df82bbb59a Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 19 Jun 2019 13:12:04 +0300 Subject: [PATCH 3/4] Bump load tester version to 0.4.0 --- artifacts/loadtester/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/loadtester/deployment.yaml b/artifacts/loadtester/deployment.yaml index f4a44620d..b5e61e1db 100644 --- a/artifacts/loadtester/deployment.yaml +++ b/artifacts/loadtester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.3.0 + image: weaveworks/flagger-loadtester:0.4.0 imagePullPolicy: IfNotPresent ports: - name: http From 647d02890f79b38fee1cba3e8810b575820fea6e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 19 Jun 2019 13:15:17 +0300 Subject: [PATCH 4/4] Add HTTP metrics when no mesh provider is specified Implement request-success-rate and request-duration checks using http_request_duration_seconds histogram --- pkg/metrics/factory.go | 4 +++ pkg/metrics/http.go | 71 ++++++++++++++++++++++++++++++++++++++ pkg/metrics/http_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 pkg/metrics/http.go create mode 100644 pkg/metrics/http_test.go diff --git a/pkg/metrics/factory.go b/pkg/metrics/factory.go index 986ffb42e..cf669729c 100644 --- a/pkg/metrics/factory.go +++ b/pkg/metrics/factory.go @@ -24,6 +24,10 @@ func NewFactory(metricsServer string, meshProvider string, timeout time.Duration func (factory Factory) Observer() Interface { switch { + case factory.MeshProvider == "none": + return &HttpObserver{ + client: factory.Client, + } case factory.MeshProvider == "appmesh": return &EnvoyObserver{ client: factory.Client, diff --git a/pkg/metrics/http.go b/pkg/metrics/http.go new file mode 100644 index 000000000..9c169e441 --- /dev/null +++ b/pkg/metrics/http.go @@ -0,0 +1,71 @@ +package metrics + +import "time" + +var httpQueries = map[string]string{ + "request-success-rate": ` + sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="{{ .Namespace }}", + kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", + status!~"5.*" + }[{{ .Interval }}] + ) + ) + / + sum( + rate( + http_request_duration_seconds_count{ + kubernetes_namespace="{{ .Namespace }}", + kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[{{ .Interval }}] + ) + ) + * 100`, + "request-duration": ` + histogram_quantile( + 0.99, + sum( + rate( + http_request_duration_seconds_bucket{ + kubernetes_namespace="{{ .Namespace }}", + kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[{{ .Interval }}] + ) + ) by (le) + )`, +} + +type HttpObserver struct { + client *PrometheusClient +} + +func (ob *HttpObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, httpQueries["request-success-rate"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + return value, nil +} + +func (ob *HttpObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) { + query, err := ob.client.RenderQuery(name, namespace, interval, httpQueries["request-duration"]) + if err != nil { + return 0, err + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, err + } + + ms := time.Duration(int64(value*1000)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/http_test.go b/pkg/metrics/http_test.go new file mode 100644 index 000000000..a0d31cd76 --- /dev/null +++ b/pkg/metrics/http_test.go @@ -0,0 +1,74 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHttpObserver_GetRequestSuccessRate(t *testing.T) { + expected := `sum(rate(http_request_duration_seconds_count{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",status!~"5.*"}[1m]))/sum(rate(http_request_duration_seconds_count{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"}[1m]))*100` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &HttpObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100 { + t.Errorf("Got %v wanted %v", val, 100) + } +} + +func TestHttpObserver_GetRequestDuration(t *testing.T) { + expected := `histogram_quantile(0.99,sum(rate(http_request_duration_seconds_bucket{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"}[1m]))by(le))` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + if promql != expected { + t.Errorf("\nGot %s \nWanted %s", promql, expected) + } + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"0.100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := NewPrometheusClient(ts.URL, time.Second) + if err != nil { + t.Fatal(err) + } + + observer := &HttpObserver{ + client: client, + } + + val, err := observer.GetRequestDuration("podinfo", "default", "1m") + if err != nil { + t.Fatal(err.Error()) + } + + if val != 100*time.Millisecond { + t.Errorf("Got %v wanted %v", val, 100*time.Millisecond) + } +}