From 921ac003837245d35d0bce5c90c1f9927a0c50df Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:33:00 +0300 Subject: [PATCH 01/10] Add ingress ref to CRD and RBAC --- artifacts/flagger/account.yaml | 6 ++++++ artifacts/flagger/crd.yaml | 12 ++++++++++++ charts/flagger/templates/crd.yaml | 12 ++++++++++++ charts/flagger/templates/rbac.yaml | 6 ++++++ pkg/apis/flagger/v1alpha3/types.go | 4 ++++ pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go | 5 +++++ 6 files changed, 45 insertions(+) diff --git a/artifacts/flagger/account.yaml b/artifacts/flagger/account.yaml index 9bbd84107..0fac89ad3 100644 --- a/artifacts/flagger/account.yaml +++ b/artifacts/flagger/account.yaml @@ -31,6 +31,12 @@ rules: resources: - horizontalpodautoscalers verbs: ["*"] + - apiGroups: + - "extensions" + resources: + - ingresses + - ingresses/status + verbs: ["*"] - apiGroups: - flagger.app resources: diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 134020e63..61eac4087 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -69,6 +69,18 @@ spec: type: string name: type: string + ingressRef: + anyOf: + - type: string + - type: object + required: ['apiVersion', 'kind', 'name'] + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: type: object required: ['port'] diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index aad996b89..f189a31a0 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -70,6 +70,18 @@ spec: type: string name: type: string + ingressRef: + anyOf: + - type: string + - type: object + required: ['apiVersion', 'kind', 'name'] + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: type: object required: ['port'] diff --git a/charts/flagger/templates/rbac.yaml b/charts/flagger/templates/rbac.yaml index 95a2f2498..e03755c08 100644 --- a/charts/flagger/templates/rbac.yaml +++ b/charts/flagger/templates/rbac.yaml @@ -27,6 +27,12 @@ rules: resources: - horizontalpodautoscalers verbs: ["*"] + - apiGroups: + - "extensions" + resources: + - ingresses + - ingresses/status + verbs: ["*"] - apiGroups: - flagger.app resources: diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 4b0b9d129..e9eb9a49c 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -52,6 +52,10 @@ type CanarySpec struct { // +optional AutoscalerRef *hpav1.CrossVersionObjectReference `json:"autoscalerRef,omitempty"` + // reference to NGINX ingress resource + // +optional + IngressRef *hpav1.CrossVersionObjectReference `json:"ingressRef,omitempty"` + // virtual service spec Service CanaryService `json:"service"` diff --git a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go index 4bb9551cc..dc710448a 100644 --- a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go @@ -205,6 +205,11 @@ func (in *CanarySpec) DeepCopyInto(out *CanarySpec) { *out = new(v1.CrossVersionObjectReference) **out = **in } + if in.IngressRef != nil { + in, out := &in.IngressRef, &out.IngressRef + *out = new(v1.CrossVersionObjectReference) + **out = **in + } in.Service.DeepCopyInto(&out.Service) in.CanaryAnalysis.DeepCopyInto(&out.CanaryAnalysis) if in.ProgressDeadlineSeconds != nil { From 5f544b90d6dd14d4a4c342571b434298829c011f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:41:04 +0300 Subject: [PATCH 02/10] Log mesh provider at startup --- cmd/flagger/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go index 9297577e0..0c09b7dc3 100644 --- a/cmd/flagger/main.go +++ b/cmd/flagger/main.go @@ -99,7 +99,7 @@ func main() { canaryInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries() - logger.Infof("Starting flagger version %s revision %s", version.VERSION, version.REVISION) + logger.Infof("Starting flagger version %s revision %s mesh provider %s", version.VERSION, version.REVISION, meshProvider) ver, err := kubeClient.Discovery().ServerVersion() if err != nil { From 177dc824e3f4229ad9796b224bb042928209313f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:42:02 +0300 Subject: [PATCH 03/10] Implement nginx ingress router --- pkg/router/factory.go | 7 +- pkg/router/ingress.go | 165 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 pkg/router/ingress.go diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 9d8f66ecf..ab836d318 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -41,9 +41,14 @@ func (factory *Factory) KubernetesRouter(label string) *KubernetesRouter { } } -// MeshRouter returns a service mesh router (Istio or AppMesh) +// MeshRouter returns a service mesh router func (factory *Factory) MeshRouter(provider string) Interface { switch { + case provider == "nginx": + return &IngressRouter{ + logger: factory.logger, + kubeClient: factory.kubeClient, + } case provider == "appmesh": return &AppMeshRouter{ logger: factory.logger, diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go new file mode 100644 index 000000000..7e9513e0a --- /dev/null +++ b/pkg/router/ingress.go @@ -0,0 +1,165 @@ +package router + +import ( + "fmt" + "github.com/google/go-cmp/cmp" + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + "go.uber.org/zap" + "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "strconv" + "strings" +) + +type IngressRouter struct { + kubeClient kubernetes.Interface + logger *zap.SugaredLogger +} + +func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error { + if canary.Spec.IngressRef == nil || canary.Spec.IngressRef.Name == "" { + return fmt.Errorf("ingress selector is empty") + } + + targetName := canary.Spec.TargetRef.Name + canaryName := fmt.Sprintf("%s-canary", targetName) + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + + ingress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canary.Spec.IngressRef.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + ingressClone := ingress.DeepCopy() + + // change backend to -canary + backendExists := false + for k, v := range ingressClone.Spec.Rules { + for x, y := range v.HTTP.Paths { + if y.Backend.ServiceName == targetName { + ingressClone.Spec.Rules[k].HTTP.Paths[x].Backend.ServiceName = canaryName + backendExists = true + break + } + } + } + + if !backendExists { + return fmt.Errorf("backend %s not found in ingress %s", targetName, canary.Spec.IngressRef.Name) + } + + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + + if errors.IsNotFound(err) { + ing := &v1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: canaryIngressName, + Namespace: canary.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(canary, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + }, + Annotations: i.makeAnnotations(ingressClone.Annotations), + Labels: ingressClone.Labels, + }, + Spec: ingressClone.Spec, + } + + _, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Create(ing) + if err != nil { + return err + } + + i.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s.%s created", ing.GetName(), canary.Namespace) + return nil + } + + if err != nil { + return fmt.Errorf("ingress %s query error %v", canaryIngressName, err) + } + + if diff := cmp.Diff(ingressClone.Spec, canaryIngress.Spec); diff != "" { + iClone := canaryIngress.DeepCopy() + iClone.Spec = ingressClone.Spec + + _, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) + if err != nil { + return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) + } + + i.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s updated", canaryIngressName) + } + + return nil +} + +func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) ( + primaryWeight int, + canaryWeight int, + err error, +) { + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + if err != nil { + return 0, 0, err + } + + for k, v := range canaryIngress.Annotations { + if k == "nginx.ingress.kubernetes.io/canary-weight" { + val, err := strconv.Atoi(v) + if err != nil { + return 0, 0, err + } + + canaryWeight = val + break + } + } + + primaryWeight = 100 - canaryWeight + return +} + +func (i *IngressRouter) SetRoutes( + canary *flaggerv1.Canary, + primaryWeight int, + canaryWeight int, +) error { + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + if err != nil { + return err + } + + iClone := canaryIngress.DeepCopy() + iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + + _, err = i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) + if err != nil { + return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) + } + + return nil +} + +func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string { + res := make(map[string]string) + for k, v := range annotations { + if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + res[k] = v + } + } + + res["nginx.ingress.kubernetes.io/canary"] = "true" + res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + + return res +} From cf3ba35fb98835fd678e8de84fb48d7da794c861 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:42:31 +0300 Subject: [PATCH 04/10] Add nginx ingress controller metrics --- Makefile | 6 ++ artifacts/nginx/canary.yaml | 54 ++++++++++++++ artifacts/nginx/deployment.yaml | 69 ++++++++++++++++++ artifacts/nginx/ingress.yaml | 17 +++++ pkg/metrics/nginx.go | 122 ++++++++++++++++++++++++++++++++ pkg/metrics/nginx_test.go | 51 +++++++++++++ 6 files changed, 319 insertions(+) create mode 100644 artifacts/nginx/canary.yaml create mode 100644 artifacts/nginx/deployment.yaml create mode 100644 artifacts/nginx/ingress.yaml create mode 100644 pkg/metrics/nginx.go create mode 100644 pkg/metrics/nginx_test.go diff --git a/Makefile b/Makefile index 3f656da46..262f98b9d 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,12 @@ run-appmesh: -slack-url=https://hooks.slack.com/services/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 \ -slack-channel="devops-alerts" +run-nginx: + go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=nginx -namespace=nginx \ + -metrics-server=http://prometheus-weave.istio.weavedx.com \ + -slack-url=https://hooks.slack.com/services/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 \ + -slack-channel="devops-alerts" + build: docker build -t weaveworks/flagger:$(TAG) . -f Dockerfile diff --git a/artifacts/nginx/canary.yaml b/artifacts/nginx/canary.yaml new file mode 100644 index 000000000..13eb15c98 --- /dev/null +++ b/artifacts/nginx/canary.yaml @@ -0,0 +1,54 @@ +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # ingress reference + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 10s + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # NGINX Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: request-duration + # maximum avg req duration + # milliseconds + threshold: 500 + interval: 1m + # external checks (optional) + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://app.example.com/" + logCmdOutput: "true" diff --git a/artifacts/nginx/deployment.yaml b/artifacts/nginx/deployment.yaml new file mode 100644 index 000000000..814dd9c2f --- /dev/null +++ b/artifacts/nginx/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: test + labels: + app: podinfo +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: quay.io/stefanprodan/podinfo:1.4.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9898 + name: http + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: green + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + failureThreshold: 3 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + resources: + limits: + cpu: 1000m + memory: 256Mi + requests: + cpu: 100m + memory: 16Mi diff --git a/artifacts/nginx/ingress.yaml b/artifacts/nginx/ingress.yaml new file mode 100644 index 000000000..5cb6b8269 --- /dev/null +++ b/artifacts/nginx/ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.exmaple.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/pkg/metrics/nginx.go b/pkg/metrics/nginx.go new file mode 100644 index 000000000..7c6eb56e7 --- /dev/null +++ b/pkg/metrics/nginx.go @@ -0,0 +1,122 @@ +package metrics + +import ( + "fmt" + "net/url" + "strconv" + "time" +) + +const nginxSuccessRateQuery = ` +sum(rate( +nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}", +status!~"5.*"} +[{{ .Interval }}])) +/ +sum(rate( +nginx_ingress_controller_requests{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"} +[{{ .Interval }}])) +* 100 +` + +// GetNginxSuccessRate returns the requests success rate (non 5xx) using nginx_ingress_controller_requests metric +func (c *Observer) GetNginxSuccessRate(name string, namespace string, metric string, interval string) (float64, error) { + if c.metricsServer == "fake" { + return 100, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + return *rate, nil +} + +const nginxRequestDurationQuery = ` +sum(rate( +nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) +/ +sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) * 1000 +` + +// GetNginxRequestDuration returns the avg requests latency using nginx_ingress_controller_ingress_upstream_latency_seconds_sum metric +func (c *Observer) GetNginxRequestDuration(name string, namespace string, metric string, interval string) (time.Duration, error) { + if c.metricsServer == "fake" { + return 1, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + ms := time.Duration(int64(*rate)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/nginx_test.go b/pkg/metrics/nginx_test.go new file mode 100644 index 000000000..e2a93c5f1 --- /dev/null +++ b/pkg/metrics/nginx_test.go @@ -0,0 +1,51 @@ +package metrics + +import ( + "testing" +) + +func Test_NginxSuccessRateQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo",status!~"5.*"}[1m])) / sum(rate(nginx_ingress_controller_requests{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 100` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +} + +func Test_NginxRequestDurationQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) /sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{kubernetes_namespace="nginx",ingress="podinfo"}[1m])) * 1000` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +} From f7db0210ea0da32b3f3bf35fdf2b2580cf1d06bc Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 May 2019 18:43:02 +0300 Subject: [PATCH 05/10] Add nginx ingress controller checks --- pkg/controller/scheduler.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 5c16cb4e3..0fc069789 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -632,6 +632,41 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { } } + // NGINX checks + if c.meshProvider == "nginx" { + if metric.Name == "request-success-rate" { + val, err := c.observer.GetNginxSuccessRate(r.Spec.IngressRef.Name, r.Namespace, metric.Name, metric.Interval) + if err != nil { + if strings.Contains(err.Error(), "no values found") { + c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic", + metric.Name, r.Spec.TargetRef.Name, r.Namespace) + } else { + c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.GetMetricsServer(), err) + } + return false + } + if float64(metric.Threshold) > val { + c.recordEventWarningf(r, "Halt %s.%s advancement success rate %.2f%% < %v%%", + r.Name, r.Namespace, val, metric.Threshold) + return false + } + } + + if metric.Name == "request-duration" { + val, err := c.observer.GetNginxRequestDuration(r.Spec.IngressRef.Name, r.Namespace, metric.Name, metric.Interval) + if err != nil { + c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.GetMetricsServer(), err) + return false + } + t := time.Duration(metric.Threshold) * time.Millisecond + if val > t { + c.recordEventWarningf(r, "Halt %s.%s advancement request duration %v > %v", + r.Name, r.Namespace, val, t) + return false + } + } + } + // custom checks if metric.Query != "" { val, err := c.observer.GetScalar(metric.Query) From 00151e92fe360538dcfa8cbf3ace7be245b0c80d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 7 May 2019 10:33:40 +0300 Subject: [PATCH 06/10] Implement A/B testing for nginx ingress --- pkg/router/ingress.go | 65 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index 7e9513e0a..8e3f51d5a 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -112,6 +112,16 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) ( return 0, 0, err } + // A/B testing + if len(canary.Spec.CanaryAnalysis.Match) > 0 { + for k := range canaryIngress.Annotations { + if k == "nginx.ingress.kubernetes.io/canary-by-cookie" || k == "nginx.ingress.kubernetes.io/canary-by-header" { + return 0, 100, nil + } + } + } + + // Canary for k, v := range canaryIngress.Annotations { if k == "nginx.ingress.kubernetes.io/canary-weight" { val, err := strconv.Atoi(v) @@ -140,7 +150,33 @@ func (i *IngressRouter) SetRoutes( } iClone := canaryIngress.DeepCopy() - iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + + // A/B testing + if len(canary.Spec.CanaryAnalysis.Match) > 0 { + cookie := "" + header := "" + headerValue := "" + for _, m := range canary.Spec.CanaryAnalysis.Match { + for k, v := range m.Headers { + if k == "cookie" { + cookie = v.Exact + } else { + header = k + headerValue = v.Exact + } + } + } + + if canaryWeight > 0 { + iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) + } else { + iClone.Annotations = i.makeAnnotations(iClone.Annotations) + } + + } else { + // canary + iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + } _, err = i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) if err != nil { @@ -163,3 +199,30 @@ func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[strin return res } + +func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, + header string, headerValue string, cookie string) map[string]string { + res := make(map[string]string) + for k, v := range annotations { + if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + res[k] = v + } + } + + res["nginx.ingress.kubernetes.io/canary"] = "true" + res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + + if cookie != "" { + res["nginx.ingress.kubernetes.io/canary-by-cookie"] = cookie + } + + if header != "" { + res["nginx.ingress.kubernetes.io/canary-by-header"] = header + } + + if headerValue != "" { + res["nginx.ingress.kubernetes.io/canary-by-header-value"] = headerValue + } + + return res +} From 0d94c01678b58a02ad1403d0dcf5da494554fe46 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 7 May 2019 11:10:19 +0300 Subject: [PATCH 07/10] Toggle canary annotation based on weight --- pkg/router/ingress.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go index 8e3f51d5a..ffc663b1f 100644 --- a/pkg/router/ingress.go +++ b/pkg/router/ingress.go @@ -167,17 +167,19 @@ func (i *IngressRouter) SetRoutes( } } - if canaryWeight > 0 { - iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) - } else { - iClone.Annotations = i.makeAnnotations(iClone.Annotations) - } - + iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) } else { // canary iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) } + // toggle canary + if canaryWeight > 0 { + iClone.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + } else { + iClone.Annotations = i.makeAnnotations(iClone.Annotations) + } + _, err = i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) if err != nil { return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) @@ -189,12 +191,13 @@ func (i *IngressRouter) SetRoutes( func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string { res := make(map[string]string) for k, v := range annotations { - if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + if !strings.Contains(k, "nginx.ingress.kubernetes.io/canary") && + !strings.Contains(k, "kubectl.kubernetes.io/last-applied-configuration") { res[k] = v } } - res["nginx.ingress.kubernetes.io/canary"] = "true" + res["nginx.ingress.kubernetes.io/canary"] = "false" res["nginx.ingress.kubernetes.io/canary-weight"] = "0" return res From a233b99f0b53d75bc73f12f36769903500e67ced Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 7 May 2019 11:12:36 +0300 Subject: [PATCH 08/10] Add HPA to nginx demo --- artifacts/nginx/canary.yaml | 5 +++++ artifacts/nginx/hpa.yaml | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 artifacts/nginx/hpa.yaml diff --git a/artifacts/nginx/canary.yaml b/artifacts/nginx/canary.yaml index 13eb15c98..bca0a7098 100644 --- a/artifacts/nginx/canary.yaml +++ b/artifacts/nginx/canary.yaml @@ -14,6 +14,11 @@ spec: apiVersion: extensions/v1beta1 kind: Ingress name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo # the maximum time in seconds for the canary deployment # to make progress before it is rollback (default 600s) progressDeadlineSeconds: 60 diff --git a/artifacts/nginx/hpa.yaml b/artifacts/nginx/hpa.yaml new file mode 100644 index 000000000..fa2b5a6f4 --- /dev/null +++ b/artifacts/nginx/hpa.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo + namespace: test +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + # scale up if usage is above + # 99% of the requested CPU (100m) + targetAverageUtilization: 99 From 79b337089294a92961bc8446fd185b38c50a32df Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 15:44:28 +0300 Subject: [PATCH 09/10] Add Prometheus add-on to Flagger chart --- charts/flagger/templates/deployment.yaml | 4 + charts/flagger/templates/prometheus.yaml | 292 +++++++++++++++++++++++ charts/flagger/values.yaml | 6 +- 3 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 charts/flagger/templates/prometheus.yaml diff --git a/charts/flagger/templates/deployment.yaml b/charts/flagger/templates/deployment.yaml index c23ce93b2..0df390b75 100644 --- a/charts/flagger/templates/deployment.yaml +++ b/charts/flagger/templates/deployment.yaml @@ -38,7 +38,11 @@ spec: {{- if .Values.meshProvider }} - -mesh-provider={{ .Values.meshProvider }} {{- end }} + {{- if .Values.prometheus.install }} + - -metrics-server=http://{{ template "flagger.fullname" . }}-prometheus:9090 + {{- else }} - -metrics-server={{ .Values.metricsServer }} + {{- end }} {{- if .Values.namespace }} - -namespace={{ .Values.namespace }} {{- end }} diff --git a/charts/flagger/templates/prometheus.yaml b/charts/flagger/templates/prometheus.yaml new file mode 100644 index 000000000..f1fe583b4 --- /dev/null +++ b/charts/flagger/templates/prometheus.yaml @@ -0,0 +1,292 @@ +{{- if .Values.prometheus.install }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + - nodes/proxy + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "flagger.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "flagger.serviceAccountName" . }}-prometheus + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "flagger.serviceAccountName" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +data: + prometheus.yml: |- + global: + scrape_interval: 5s + scrape_configs: + + # Scrape config for AppMesh Envoy sidecar + - job_name: 'appmesh-envoy' + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_name] + action: keep + regex: '^envoy$' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: ${1}:9901 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + # Exclude high cardinality metrics + metric_relabel_configs: + - source_labels: [ cluster_name ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ tcp_prefix ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ listener_address ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_listener_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tls.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tcp_downstream.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_http_(stats|admin).*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*' + action: drop + + # Scrape config for API servers + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - default + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kubernetes;https + + # Scrape config for nodes + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + # scrape config for cAdvisor + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # scrape config for pods + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - source_labels: [ __address__ ] + regex: '.*9901.*' + action: drop + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + appmesh.k8s.aws/sidecarInjectorWebhook: disabled + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: {{ template "flagger.serviceAccountName" . }}-prometheus + containers: + - name: prometheus + image: "docker.io/prom/prometheus:v2.7.1" + imagePullPolicy: IfNotPresent + args: + - '--storage.tsdb.retention=6h' + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - containerPort: 9090 + name: http + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + resources: + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - name: data-volume + mountPath: /prometheus/data + + volumes: + - name: config-volume + configMap: + name: {{ template "flagger.fullname" . }}-prometheus + - name: data-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + selector: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - name: http + protocol: TCP + port: 9090 +{{- end }} diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 0f1195ab2..0d1a07afc 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -7,7 +7,7 @@ image: metricsServer: "http://prometheus:9090" -# accepted values are istio or appmesh (defaults to istio) +# accepted values are istio, appmesh, nginx or supergloo:mesh.namespace (defaults to istio) meshProvider: "" # single namespace restriction @@ -49,3 +49,7 @@ nodeSelector: {} tolerations: [] affinity: {} + +prometheus: + # to be used with AppMesh or nginx ingress + install: false From 8914f26754c27da2aea9f1d59e0897cc5a896b49 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 8 May 2019 17:03:36 +0300 Subject: [PATCH 10/10] Add ngnix docs --- docs/diagrams/flagger-nginx-overview.png | Bin 0 -> 40939 bytes .../usage/nginx-progressive-delivery.md | 355 ++++++++++++++++++ 2 files changed, 355 insertions(+) create mode 100644 docs/diagrams/flagger-nginx-overview.png create mode 100644 docs/gitbook/usage/nginx-progressive-delivery.md diff --git a/docs/diagrams/flagger-nginx-overview.png b/docs/diagrams/flagger-nginx-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..f8dcaadcd91b0fbecba555a280aa50babb75d693 GIT binary patch literal 40939 zcmc$_WmuG7^fo#m3PTPz@cLDNK$-h}?`473?V%sP*NQl2?`T!@>~Brh zy?MX2^f}Ji*~vC)eW+Yx!_&?nvW+5m)D8q<1Su=X>H17<%n~_NIa1$x6jVpF%cU8{ z7<_&ybCO>)nc?)jv~SF{_50-5RC~jY`1Z6a!5kKdo^gi_1VTbUvUpe^2pj|gaISyE zj9mSAHS%xr{~Y;$7XN=$@V|@yf7<&e{(tk}e^>DTvwKKL0QUch)T79SsvLMgD=b?)B8Ri@^<6^$nH6S|b#}aUSq}D~7519|PLcxUW`jEE zqHinR=0)>l*{Oo{JMy@;4OKZjto-O;Yt$F7ULJXp@l|jNYhTxnqu1xfBjT|Ixvp0wh@Z)5Q*zy0*2jnA-Hk?n zHs98|UQYkjdo4)*@^PL;HhGfH)oNXd9gC%j%ZJKIoHv(suf#|9MBKfL?+-F}s=P9c ze+|J_^Md{O@t%z&4%?mcO1)Cbv#yc*C2eXPcXO@*8Oa*hfAU6L2fwfIDk>(C)El8p zR6b_sb(o=|tD!39ZXS~Kl)-y@$uo3tj%(a?^=5qz^P^21S-nFvuf5Bs2(Fux{eaV~ zt8K0Fisst(VqGBU`8;?`VwOz_+sutsfg=fw_3;4V7TdgEY412b)>4hTPZqUdKy~^DD9W^YW+W>TRj^CPB}n7QyQv zF$p6|lfrX(Fh4oZW9bAwtFyUY1mJuO6U4;|*7=jVJ#gtfKL6a|kpIqmkK~n1uok`z z90~bub3bN3Dne_=Y5v5y)_C#hi-*|v$qq6kxvFLuUt+*uLV^Z=q{+Q3b0`g%dOp8J zoH2i#r?p)_uI{hjvZ*)TOyg*%7nY2gPi*7-XwI=XDrdB8aCY!Xr#Kn%y*7;>ufmyT zckuZ2lf)JoDN7&o(Z*bq&#srI`S_FKy1Jn2g-4Njs1`bK^_zfaZIGCilh}kSdMzF> z%1--a_lNn(oqfcWSyMlr42~-Kz{b;FCh|pt0-j;Ry11LS!-4$N6u(ad^gH)vRR1Ft z$L7@Wfi)!c_}9+^y_9dywGorBC@U(dg-?#h@P&0a!k*P?F7K<*5vN_8o{uYjasVDb z8TO}y*g(*O^3Z{g5c>#i1{Yj)QqI=#KhY3q!IUTdtB(Itg6pELI(cj5eN3viK%+07a^!fLwccSY_L7@SWr~+C z(~Q(zXWnJghybhOpe3KXwGX{+lzl09F4!ilVlvw#Byx&OY&kAfm^+##c7N*}AE6=Jk(qs&@si^ZRpzQMOt@ z;@+E&=M$x=O^1(~PDBqQiywqEoM$%#CVW9 zs`p7~kvzpco&NS%;aVTlzX{g~1hK7v8hY>CVw=;iO~?6oLeX$av9prC{c#oF!icx+ zN7ZbWhS0om$(k82z>(F8IB!P`>5=--yh*8B`kwA7 zU(ip;5T%kSrS^v~>h3geov{M!SUO?uWwB;zX{X+;mcqglECZtch{ z&Fk5=55`3{^2{Y9$6&kLLjvmo%^Sk&u@+d*Lq4Mr9455zKV%%2%}U)B9MJJukyD5^ zF)ej};9AZ;|Lli{wD2hb^wf~bpOvsiNcFt2-3LSpx{3sFs-&3JY0$uHrMVw7oWAnd zpFZF$ADHeJ>FRUcHgtNu8~*zzJpdD(xqHv#;Bn!DlwU_YDMzG762O}r8}}!Urg+LE ziBm`?iEd)=9CYIG$3%2&AF?lt`~nQDHV+5v-ew7@X7i<@*lTaTe=ofJ%jJgYbYzJr z1jlZIz3LT$8h|0cUk11ZmOI^Dw1VG?)s;$1?+*GFPN_FrwxvJ6lF@>u>l_u<=47(O zUkNKa2oW$=IU#Y!9F0I1H%THuF@{QfzO`h6+i_LT3=P7Z#WNc_1Ku^x3!w=I(5_?= zX;lx)*mYSJJI~5 zS)LQw=_7&XTo+2!xQEPOutOGy5 zVNB#>3W)FYT0};L8g``c8-ZgjJ;<3Cwk89f*6ls`&fE|3YaBPjxmcC%vf+S0-%07I zv0#}#bM32nMQu-cKu#j(Q$>>XG^fJA2y?7?z~-s38lWHgi!x|N7fmEoH*+V1|D3;< zB0MturYBmE{_PK*d=wY_NBqwu0iURk?)0hU@y$2H0PWGVu7|4&7pRJqJP%u$z!Qg} zVn37vC0{7S2LttQ0m-EscQ4vMY#&Z~4n-ws=vK~cA~v^#gq}}qkNQV(Ap20aJsw+$ zPkSbm6yvgqITnj9E4eKwzJFXP=;Tf6My!D#&#Hg`wV%eb{5obUQ^+&h!XZY&m38xU zC4@@P^pD5_+2F+|=8xwVvJ4dPBmoe}p9&K!cP3+!N=?)!=x(q*Md95+mj z81jp>aJoyOZC1337K&Xxt`Q#KzViL&87NV>1uvX)lZ$AE`PrTJPd}4h(RWb`hBUZY z7rb%OgRUKqhqShyUC4I?a!*CXe8ctOPRqyajRfJDGi<>IfDEG1<*j%!K-(7IScPnGSCE&3@c*Q>u;CLIWjjQ)|11xuws_*AE|N5n6h-n_%hZW!S z2I~Ut)REUKbJ!Bn&i)+!q)cI}31KslK8Xf{qJz$Y``txSz_?&=TQ(7E0qOTy=B8F& zvgGGQuzTT`w>~dnOZa`#_x)D!jV?zXwjwF$U=3$d$bgQ~_}wGVt__pLWVE6^cJVV= zrLj$m{mZ&anAgDVuR~s;4&_R=y!-P{y-8#6${9pP)wDkv9I83BK$&zFJcK0#Ev!h1 zssleZ-P_Il4pvt1ZPrW*L(vxOOP~35jPBw$MJpj;@dObyxwG25w?A+KLJBoUcwwlS zgY9N0eqLz8I)Ev6ADUE5S;#any3T;RZ9bFL6;??)ytu9NPUG2Vi|^)TYqI#gDpBI% zG^yM%tN6sRo6ui7X(FOKsrjPrS@3GZp80ojkpkE|++3H!AND7}NHecPm& zxgm4xxkU1L)u%~%@->>#I_=#~sz^~9LEX<=)HYGCGRS~^P4;STvsgR@Orzy{Jd6i{ zMJi?djLJP0SXCQs+B7~VTc^8!Geq@MRz~Z-ClL|5i`#!m4sec|+j3`z=ynmvoG)*Fv4Kk$F*+TEYl zWOw%sSa|OBFKJedT(Lo_J5q-@+@F;ejPC>r7Bfl_GZ04?QaI`d`|Gny^1#FExY^E} z*xA8QcL0Y82lR5h(i#Wl`!q>dY4{b5Kcx2Y%RHM!&!FF;{-QX8g@h*+nm+o-iWJ^d zF;VgeoNh~cT5M8mP;2jm05;OyK$+)5*&J?+PhD9PN$mKA#D@CY_F(zD)lp_WC=!C8 z0tJLG?Nd^6xCi?~Fx2xuixxPj!^0(xW~+=ZAB%Gt{5I8LY?p4-qi#PDH%Os@Y#s6N z+-u7!Nb;)DyhtW;d^WjkUgUM0U9ncNzEdu1Q~k$ zw7?UK>?T0M?;O3;{k807S5sEQ(O}X?f0U{-E+Ren5ckJ|*(<{Q$C7MslVB~2 z&5*T);!M`y0^}D$dA<4W1DhcOzI3{TR*=YUvl+jm4M=GwqI&e= z5x(yu9MOC*(=H4T4iszN6|fS?55cTpymi+ykleT($S1+|2W|Fvmh%{k>EUrp`}Gnx z>d<0EzI+G8$$}WOv(r>fPk4o5TH?Y>iC7C`rI)>q)`r6oIi@N$hKf4m&xLp%5Ie~T zzcmqoI}f3JbIXo`^!_k&soSFQra>x$?BvxdKP}pg_Nd;L-E( z4;NHrrL;eALc7V@s!L6t+(N>2voyC>uES!yRpIBAr*5^k3RKzSBqft}bwJ{kpWKiA zDm-^65yRu4uHrE?6B%+>ga5p|jjZY&ub`lC29f>r2Vz6f2 zj(3+fh9g9SC3Z#A2r?3oX)&f>n?P8vBjlJH9J506D${ z-IyGvAG(O<%iFwYLW}*V_qQI!yQ>(7gtE`J^vdnZ$@+fMTR z5g_^B#L)tXr#*~oudH5}F%Y{R60Ge5iTR)9SgTU9{}5zxop=7ZdF1C&^1SFKm7axu zR+ZaSxRLHBA?NkPo^ba=4*i|*GIeW37N1}EWG0_K%>>NaFJ_ok!kB$F3j)brVlu|LT> z$iBhLj%0J#V|zoyO9Pz@|FKvJ(!ewTh7dcu^c^zpW~eiyH!);YjolJ;P+P4RS9afavskb{ZKwBBBUA?oQpRE zYxv1o?0Vxm+MV)K=VdyP7nXMQ*3Ser&bVgGhxA^z<@e4ng2lAh)1)Y`4oTmC_AvUd zr}^$j7yeqZO@CJQ;;3Zb{2WFkqFbo3p)M&dMInBk{v^j{&M~y_MR-B5a?*v*kKzrJ zmKAfkmQ)9`Q(g^p*#CgAn{{?BzRfhv8EAr`;B;gMLtSZUYE-#)Zp}BMN}1h<;$QFW z&Eyru`Ap1cJ%L&Rza29u>fv@hdwu*f2EOnCGJLkEn^y<hs#$cK+&PW zy!&l;Yz?&C|M>UqN6LjZQjJTVL25XBfnlI<`uis*CobK=94~BlwkfBUi+FEHlv6x; zIG@`W8Nr3_JG*yPB~z+XB2(JH5awle42X=3G6!FFP<;Wd^>4d=DJYX!Z;X1yRzWz z3(v+2ns|(bDWeUX#w}Q~&)sJYYciV}r+)hdz4+0mkk%y19xxDy3NQ^TH@(uRl~}N&a<8^gE=wid#Z0T zpam1E5QZo*YEA318^s43Yu+dy-Gv8|M+NJj>B7nnkP49h0#kfo&l372LLqcyEw3SI z^j5MZj?XVd4T0u)9p9{ewfVeBze`v_iR$S|;(oFD`0H(k->1;1CrW<$Etp#!X4n0j zZxUVWWO7y~z0l@xRB1(3uJ1Dy--J=A`wvp;JiK6Y{k}df=ELy~A_jtuqfA(28rlYW zv$pS&MVRdC4!B!31WWo%7hK1MWn`+PFtv6e4;)=*Z#uU2b_XX|FX3nrp?Z2Fov@La zWDl`xsnWOtOMW37Sy$NzJ=R*bb1*&eZrPPGSy-*B%p@4W==Hdqb~t|eJkiGkYJ)(S zWT0k4;EREU%cvrqG>5=Zge!#F!9{~y&Lz`~xopB2>+ z?KDNSsdui=*IUZAb#nQ)v3QGY;~u=Ypl8hx9z8pMzo_u3FBq+6mMoIs%dWG;x!$9- z)P2`32>>d6;a>Fr)M04iS&xnff5r|hS7q?5y9kl>K&+e1JF9zV{Z6>!-mnl1?NEWH{pe=T{(Vnm(I$| z;57vkj@`x2jz!VF@FTkatd^brC{$dzWEaRv83-~mB-J(%2kesli;3J)p5p9CK-pL` zTyl!^RV5q1LOl1RHPB`M0WeLTmH3%)wu$=ri5w3f)8fKSyOwh2fa3#>O>Z?}Y#+L? z*xzrYcau5|DXKMtJgqWNQvW4I_;Vzv=|m(`BSCgF|4SI(W|&!QjJlmR}w&C+KvKVE2#u9Ydx!_Gxl2%9mBDi5mmJj z7so_aekl{}6S)Qk$ajz~E;kh3e*NRNklXH8xAiK5It*go`1o^`c;;DA;JAHmg8VZ$ z-j$aSzMcoSl-1_XAUD48KMne=fVQ1ttbPmH*CF|`Ehlwt5)}G@AjYgS`kub>uS%MG z9QSp!Mw{>qYcQ+}N|>=t%kYDRtyKJt1Ef$aH`5oD+pLE0tdfs30-mwMpFXSUEvN}7 zm7mLxE4cK$jMa_Yxb_kx8~5>iuU|6t>% zX)qk32W+k~_GodznrI(V<`Y$+@jwju;bUajyGU%zA)1%mdXkW#FR6?|pdrs#=692` zx?PK+ZQ7a7b*Q0`m*vlHKfd*>eJ;Y)Qh~&XPE?J-ne20_S3=;yWD=>>=;c^4|dGi_FcIcF0bx5xH3k5LawNm2Z?NwbT=Sv zYAKDRJX4>%ue;~@Pte(pP$}Ce)?Or)yP7kXTP;2=LkSToj3{`$!~9g$`UskcYkRx7d zp{usB-Nmtt-A-}`$=041GAi0Gf1Ja3hnf^vvRuAj$+pIS+V-8m<{I%$eQ?>0492Ro%Nr-wS*x?QRd z4a62z7P5PEqd0fYPyvd6|JsQe^7Mvja$pmrH%KtWThcpjSg}>A74zS>07iLMuZY6u zg%3Bj7}>~)Z(pk=rSW1yl;hO*9pw{Db3^75b=!`0O<_~)E;y(?=!(#+!id-Eb^%9WdS0AU z8;_}y^FG~trsFim&(^d}mZm9ca^=n=C|2U=gOiPT`|7A4Jl~DYR@t$mHAS_r=XwbI zN`&!1cc?KrQ7e=l=9y*$y~M*@)%-brB#vT%m~b#mIxej!B?du&D_E`btUdoqwBpIr zV!m&{^f75J-jW+21x$+qJsUI8#|4FAnh+?h(h1Toa|3kcg7uR?5Qu3pC!n+#F}y!- zs2|TCE=bwhtJJ#&0+G6LlV(3Kn`|j?biEC$-B#Y5le#>2ox>u;^l;QKj5Xgxbswwr zyUS4pB0LYXUxod?{W*8CA7}UMUjXYn@byJc3Z8=jVZg%5(LN_R-NcI`NL|#e; zo<9d?9uYQ1Oc=Ow0d?R2dh&|y7t3@L4P=f28 zJcMq?)QbBG613%>&^?A&M$`K^0rdVvYIeAoF^QlVTb9uM!ae^x%{Q(WT8RmOvfmSl zCHxN}=Q_Q;Pc;JzlP(djzg+6Nu)u(W~D+XDK zi9aYAy@w~T_+{M8<_U^N7s1-!XZZLpu*OWVL}SKZk7U#xiJ(@@YD0bBx}F-+y}pfM zscQ5yG9pV{lN>m`<|F0VKFT588aWL2suJ*`Q(xixo+9kJSC?de6+&zf0?i#4_otc?s~)rOs9=3tt7c_j+k39lx4h0=(|Is?1G2O01#t%2(vnhoyD)7O6Ts< z+%91Oq7Cf6TErlqpwab3H=;f=3$lNR$ozef$*duVh|#f z;q}deI19jC?9MV*=*ilbCqj$Ia=^F}8?qhKWx%`01Z@CFwf16D!0jFfIgZoKgMll* zJE#~;B1oTvw*QpYkXw+u$sq)jmGAU`t6#AS)s8bs)Q;cAY2G0}KHQ_Ni)(E9a`F(~+M_iQCun+2vccAIqa+8K}F81+r%WY8sk zym4j>-!ci{%Z?u(5NzQS(Db6!(cdGyMBuFTA`$?}HLE&KrOL>W3aj<7CII+JJq|vx zR%%@LGA#z~5&q@Wg`kqaLzbIj)44K_VaUYH$Krta(G@%$PY~~udpz+l=y{Le%b_=C z7PrhS;{gkF63wRh+(W;Z#>fTHsWoxU^9^bcCJO{yj(tuN=##Wa>ePl)BJe;u3Jo)- zSB7U2js8lp$m8NXtqAw=@DZal+1TBcJG!@aSZg3b5;N|w({OHo9^v})XV5?-76#EV zca*P%vd31N7(tlT$76!-O0w*8&d_apL>gCco{sqGUEUy=q!DsdC*Vlyf~@dt7`K;~ zc(8n^z8>Ajfa@0T*Y1@m{(V4DWngA}tbmb$s}cvG^?Y8nDa=9{ z;ph;|0t|LK+~NJtgLx>l+{MduO%d+(6c`od6q@1giT~9=Kwp4f_^-NqGQSUKHH++9N@M|(Sq>6OJGE-&!8()R=5^|}jVuvQtF zX3IIb&S5i2<^Bk`dz<6L|9VL;zmatP0Sk~lW2zI2`ZXl^I}#IXYXBF()>x zpck|W+$rOOIj5sC3Lnp^BWP?V+I!Yo-iaTv0g5l(G!u|AS+$7IYKZ_ zf(@qgW@>($yetzb1j`K-oU_)UXr_Sa`nK-+sD|2Osuu+FpS}O}H+?TR`YY!39%e6k)|2hk{pHVnFu%>hR#NQg7PYJD>A0DKSE`?gLMWZ7 z=z?#;hOzt=$SbxR2K!9v2>3PE_ejCuJflbIpEK+53~BdLp^%Qa%}t!LGdR+ zoz_&}FJ&O3U_GRTU{rR+nNDHJn9B8JOb*pjd-p^p_R@h=dR`oL(z4M{qmhnk<`IAa z*WjLTL9<^F!JpYP+f8`~1_q&LzaGT48nER)FP(LNL#X_>`RK=^x{W<>m*bms;e_ec z5}ppuU1XjUVc<+s1zg`J--rbl_TDRzw>*v>_zD};4oBq0GKE%>_S2ZO_o}_hV{CiO z25X<7bi@A-!va_?+o~pStb)>=mT45kyyZAfX0$b6G~nl|-x`^Lo*EIpHH4mJv3I15 zNgfKn+I&zlEWo@e<=)S#5N|i~h0CaVi`OZyeNxHx{`i~FQbA*o*YsNCQ$~b5(WQOV zaq>x)yowF%dlT3WBFjuimHn@v<)M4C8$mNc?aRqDTA?vN23|u-8h_N14NWoSKw}`+ z6;sLusPy&1)yK@H8>J%EW~opRx~+9kqtWvbXSQ6=@Cf055E|;hBSr*S6s#ItHHHu7 zjFd@rgcSJi9x8HM^AzvQfbG(m9-Hg;vH+hCpXptJj)L$(1w44mRGJduM`TFq+20W> z>WE6k{FPL`L?p87r`9m0P-NlZiPO8efscnSzFGQ$_A)+S2_;O?+f zLOk$ay0fz6O2(B5)_%3yWT%e)Gn?DE!lnE>5d~xVCxF=+Vr)uZ|Ek$mXiy?q6t+tx zBpwR6R7W`cPmaf8AyUF%JI7~54v09f8BqZO8pnM5%AW*wuhbFiqc)rgiNd91?J5lV&9On}kKw#Qjill$fa6I|>dla)fX+ zr>UO*#N{O|-@pxQoRkpxs>M4_b??D7LCCZS+s@B*=?Y@o@9m${)`>goUOHNPe%+jB zCQeHAn;}76wR&^`0v(XEr*^5z0$A#pPBWMv8^RW!QY70KB$>iDo+c7xZZg$7KPB=# zY0@f~<-HJXnNC*_B}vx4YgK>3JKTSCcw@uu-0@}F)pjba2v;gqJ{I(#Nj6yGR}UuI zMpQ2%J9uh*0$?ze(8+tcl-ZH}$6Xtoz>>Ze+|g_DLRV4z!1@uJ-U>+Us-yZxAR`Q8 z)d>lMK5<6vy7d;k^mT*xK$F{{Q0=MUXf2+-A!-%e?!HRG5Fx0xYXXESM1wU>BvE>K zVfpi^z>Dn0Tj;BPxTphD@QzdPolA30>UAa7-QWAXZE;RbfCz6CE%nQpOzDX7*hNh$ZS03<8| zA;$Ua6`$= z-|!3i%DM<zfm`^ zMyOp&Z|`gI&U0$U>q_@j4Ndq^fIN$cMYY_Bh2szp^?yu5l`m8^{-%QNdAbM9(+DW@ z0$WOO5AG?ahUYb~#oN5yRu9q@7m5kTOoZPn(f`HPY8E=w;(Gcq4)s|X2uGl+$DhX` z{ZIbbxtHI8Ee8o{Y<~{mVFXx@aO}{0&=~W_%pw!2w;T+sjVVr~|AA7>qHv&&Ps0Vw zw{M-k4fGJ?WrH0lc&gwnnM)wjV#21uhidwB?7*5gn1ehDna=lLAooe3{L{JyCq+x#Q=@7EYbbZz=bctL8%6CcuDbxc(NFCwkNOcT{nm6)f2xW z{;{mo0DiPzt9ssqy9TK_NS%xRGvYU4R>mL9)PL6L4z@EG{euf8_2uF}{M^)My7<%L zFTM3Sfz#x%x2wY^kMS&~P2qnQk$72c+_$xJi^Y6-fuo`3CVMq6HlR!Yox+N58ItF+ zMH139R(-9!v-@kV98gYDVUsfas*>A%$KpifpO6c-ZrYWkkDm6 z&{@Z@aBcLZMi}ypn;X!7dUh_eJ{f9a7^^QIp)u}%P>+mDapl%|vQG@wnmkz_3Yw`* zcw72(-^}pM_CKIVqERh5EQ=Ch$fWm!od?@kYyQy!G%)Mow^UH3p1ck90)3*Vs_^2S z(2*$x#pwQ&dfy#ACVR(%KmTY~k_e$1DTVHtL&2`8BuJ8O)30GdxnJ#0Uo^!)yc<~IuCY}|42__QJrR~*ty*NE!+>3S zOP1;zQA0hhp0`Lm<2t5ZkLMcJw||N@Y+<=wolOiIk?eLh@hFBnu$z+=UsmJeW7`?M z5;)$D?~&Jb)h@v5_SJwVeiDz=c8*T|_Ki|w_)&h>AAb8!$TLN__FW&-hxgt)AbyOd z9=6w>B0tRs{B;csUafx7CjzY!YS&2CX$VG&7|W);F0!>AsVx0$3h+X%ng{Igh6=OI zx?X|;9OvZa4R?4-_3}rtv4}ELhqYAsdRhL@W12H1%AZk;dKtw8Q(i5nhfT zmcF{pGI2i!EgPAqidCu;4f_6>^re%H_hf^w`-+QaRz03mQfUw5+XsZo2;-4sZ@rxmh{95F*&``hhP^2FoC+M15dM2Jdc_n3$K zmrXT7H-09zTygSl4a^hXB11JwZnXY@!vPUZHk2D|vSO7KMl;8Td~|*FL_ykvYyws7 z2t4bbepc)Iord~jeH|@2#3(c|_|r)5dImYy8@K?OV1;<6_Z@%o^{Z?n;Aah-R@|}t zRDIoSV~`%V5VxKN$3RO{%>Ebied+PB7r*%u!D0OkyyQ-7N~Mg3+4SoLn;iJCb<sKh9 zqf;Ee<6>wlj7&wEq@E$8|CU zPKy2VZ3;D{yZ8d*)Cw*wfj{3P7YmwIUe@Z! zm%6BfpO)rZMewx+R>M%AZF{fa3&wLZs*Qu?E!N^S(mylxrZa4e0oegj2m)Xm%r`<9n0LI)Cd2Ct%U8Ax{{r z`Ii7h{D53uM2vP0qj4A(LYWdw49(F>vvd5Kl)-5vHz@>PEq=%( z`rBjRjvwhaUgD5)b>mFZmz6!>Aqn!qpZi1q7ZByIdj98$aBABo-hJ7)F4+(FzZO4d zXQ%sc5pnG>BE`JZp1CC3gyEC-meIQsPLU!gHlbgezF6p~{3}Y&zooyyJ$#lI&8eE( zqfnYsmD%z&0T0hnie~ZFUl+ow=R#a4pk!m`r`XUc#_KiMmVf@YO9!80j@*Wb@k)=!vBQ&&_CkrYN@0L_d_V|TvG`L^9z)>PZ;h+ z-M@RCGmIJcEVZAE=^ses%?OU5e||8Gh$ZgOSJhqc%|; zcqeoIOvcZ)J%^J-;fHX2sj&aOthfB6a+jA;pU9`QO$?fY*2J9q%GjTo*9nja+TMc6 zGvZ+*Wg(1K6#RNozVYj_)x0=zFCm!0Qb6x5>A*Sc&ytej{|s{TW}6gk)``SDs8*YN zMJ%)qhv4A7Vw6P>g-dMK56qy-NO1ZaAUIfXMm$JWp?3V?F9Afd_2B`E*F6^zEc;LT zb}p+522J(AS&Re>oeL$0`T5hhSu$4=KA^l$A@Hf zhfhas`~0Q$u;Gldq0W{gPJ{7JGu2M4V*F-HPn(^ewQ+*F5h>6oih7yO+OP1)B&35G zXmmg^3|V@5&pRlP6|a4L{~DeruQwRf^H%XY05`mC3?`VHD5ceri-Dka=%K*tp4qbD z1v8NBT*2DUTi19#4|?iXwl?Y+oDff(@o|3N`$VT3$Rhpz{Vg>US;CeZE1kq51yXjq zH)O`0Qs6T369#n1LbCW~eR?UPpRM+ixi1^}Nrr~0CytR|??3gL_i{eG8DCLBo{31wM+jnVouz-(;9I^D{B)V-9rx z#e4H;ETrt$I(fvZSF2Temh~Ec@k3b+-kfy$fzeb?Fu`X=V@(9QD*bKlW5TDX_fnAxTSciu=R?p3%G#pD#+v2)P~-_ZzhX} zgSUI)e%BpN?(+~g^s77EDxF$cPsjNxXc=npk^6C_r2Capk%IKgky_ zwpqhe=8}yP*hRCFEg|=$YR{Ii3>yqZvY^xz@0o6IGn2S=wI1~YxjVnm823SdG*{M4 zVeHf+yC;d09K`mS7cw(g`T+oBR)SY{Aa;9v4sW~e+f)}ftqS?)+mP6?_uOkZb0J6 zkkp^Qf+&*s;t>VD!O*rZDW0nMc#}qQ^%$2}tVr@9nE0{a!UIt_X}D6tSRWo%Z&@Zl zI=`+#rg!_MJ%O*6q)aVQ*xxp6 zX6rK`WM1g{nC^~1`Q~qAmKP%8s#sf?KTgExT?iVAasTFDF7IhLE2L#Gz8R*mpX!+? zhgw%&^->aFG?oH73HB<;sqbR`=nvCRe@Ek1KJGRk0i~AJ1X%Ix0VsFm;={Gt9BwQi}d$GrNlM#(ThNrpTkgEK~|N zxVfEdsF55+^okAVH3+yUT>PUGV{#)t)f%cJ$5XqIvB-L;l zOJ4xnRc~Kj4E0+ryaE_D!Ieb(@(I}g1;^MXMp6a)o<@9@J>K7-Q}riD3h-iDUNSg3 z2ab@fsKFy7s!|n8k>)RUf!cy>RxYPJsS~9VLu8haMVI=f2no-P`Zn6V?hF)%>MM&E zG?G_5-F*qi4r1J;E^qCyndLBp9ED(guLD4e$r^`9HFJDMqHBd!ew9LWv?=*Q>ybOi z30K`j3)g}$sm%?wU+$8?eggKR8`nA5V36-z(dZOeCq6=|$)RkFI%pZyWA#?y$wokD zdO*P%?09d1JgEsweh8+j?E}x(LA&4>KH^}c#lB9aw7R2J^%g`U^hND{$;RpX@SK1y z7LlV;^6uEZg6OnorjGz?91S!hAG$Zsh3Q6O8!~wIQ%%e(G}}h{cH^BJ*Gl}y3vFhx zGEl5m$Z}h`k%E@;jBjQJo^M6zZ^^vxv0H_l?31;U3A~hhmeA(3qfM;=ocv#8eR(+4 z-}ktzGq%RQXI_joOSX{xWgC&5s3;6V_7F*8Xb=Wjvqbi6NlHjrhipl9iELT3XaBzN z{ye`we$UhMnCEuyJ?lO9+;i_aF9Yo=*7_X|0M(=2@E=2O+ok#JD*;_ZyShChxMcYU z0#~@qiwM}t7@hQmA;thQ;O7I7(5Gu7hCMiF-Vej;tdgJ6pvs%(L(xSJ-ycPaQTQ&e z_F)zcafZ`$X7hCUIo)B{??_a*K(6~`LXPf}Yw#sw#vhNtzHf|`k6~vD#hv}7>tlT; z{)##E-4;%I|Ngh;PY^2@6?&$0`l-0UH7tH6x$*gDv84=l^>u@gvvsyE-;M`Ph9Ss+ zx$nkt+Ib7feqp&kCY9grR4!OgGhqKsL6@gOczJ}EVzF@gf#Cn3yT^oKZr@uh&9EfWybv1p@6BBg+XPyp&E%DYY0cq!@1a-n2GdHGj61-*~d-@CM&1Aju`+0boWeJcIA`#Z*yR!00^@FO(d9{imF{ zT^B=DfNgP_KZ#$mGl^ex5!)d7;8ON(xYooH+|o{{6N{M41#^!$;QNOD{f}jLsX#qh zv+tD`t*#io{{5f?9+EaUKk%6Y;$yX<%gwDC4y z`qQ}&ey!;)%@QB{bg{S?_U?la%E*)d;H^_WEq9SlyoQWTA$0UI^|qw|r>b{(TwM9D ze8%2hgaa2v!fFul7iIBw2q!>Q4fgKw<1{~Hv%lnZMk6`#*B|l=0o3d)8P&#%Y(#A~ zc@jBDyl_Oa#1nG5UT(iWau`=OK$o9a^{cCTv$SPz#v$;zkL%;Lq$y8DbYTv0v?do9 z`|sjth6RZglyM(3n0CYtgYNksv8y`N0aImz8Sb=|a)s~ID=t#@+P*#{m&-8qU z1H-Z^dSjYL>{@XM$Ne|ug`PRXH|tdmtK}1Oj%3`nSzOt}DSS=n`oj~EXRG&i$)^)+ zZ~@)JN{{N5xD=7rG_w-8^swNf%+G(5a2pp9GD!wd5711bSU_pJA##e8EQX)NV=w2_n*%M!t)GHo% zd_g=w5%9s(_~=_NH+Kr>9_(hi@*my*(^@u4w4U0BG^y626$1-8Z1)}G`57A32cwI> z3))Ztb?s3p*I`V*h5(dW?eTkmkZ|Q>z1*dEDm{SbfBj{Px2%Wvy#-R_wsZwu?N*9$ zL;O~fB1Y%_n%0#=`AXpYZx%W!{e4`Og2>>5wVXGJJo zN>!$GjMZ9Hg#k_jS4i@U($LOO*=KA) z1g?Knw(VlkjAGw?lThAkflz49nOD2gPP0$$>cCsQ(pUE=b?Tfy#zj~tJ-IRcWSdqs zNb@PZ`L2z@xGf=O$I#W7&DZ6e-0l8xUGL1mk5)a+aHXww_npes*D0U~t~nsZkAz9( zNt_nd&u$5-t?$Mpyfj!;!#!@FXzkaxS{SFo1hA!V83{ZyDaT70F0sXmjA1U4oP1+? zf&eO5^7vT$LO_F|2j_C!E0;%dzwg-pl)b!EQ2YAf+ttFjkCa+3`$}yd>Dj8;U6<1( zjVt817$TRdf=Ou!K^zX!5*x9rtHr89bqh`$x*iPfp+z!k@0?dMpT_R_TF>W~$tCr# zWa2ROPRXM^HaHsJ9_LnxFblJmf312B8CawGZjAxn_kk#iHyqy8>_ehnT!|A>r3ZQj z=#rwpfB&SYJo)5H4AS}W`3JTwnQ`wE=(q=|M!RE-5+2B2i+!?g-Xj}^v3FvToe{ztXJXfo%^ldn z^foou1{kc}9Yi0dLK#galWH(JAJ7~*%^r?_O9MzU@c6lR9?BxIdylVSz(~h?ZKr+= zUY2R2z?@%#C%Jqy6Ty6iP}%W(L>c)i!BUpW-uqU9h_0q${VmHsacV-;$m`}`I_cLW z0XCHdW9NB`;h0H&x?|<6Ie3H&(}hrc)UadrPs7Iwtq;Jmq$e7yUAUX`<0+!B8q)bG zoz(y6tFEM7ltjkK8UGY9s}_;^Axhkn{NA&PhA4_O2xWraZX3EL%HnLxW>Q)$nZJI; zugv%ntClKmoVxs~g9Wcq%*>HrCW78Y7_W_-zSI4hQ5UxRTP8YAg- zZ_c66hGK(_yz(N#HX}Xt&+kAt{lBO7;A5#NpZYAfBva)w-0}&cvh&Nv#KQuf9ZwRQ zOe$b$NjG~q*o@z9)9A*s4f_+JR1%<$HFc83QXu!+l5s>+P5-o3$(~~tROdvNDd;>9 z=7w-&cv>>1cFg(!pIm}P+e)$)f1==UulBes35?Mwd^lD+1HAXeTb*-YrtO-)Fy=c$ z{Dn}3-B<8iA8#ab`Hy@nSt41SnA@pf=(55LymAoQyZGj7sbkm_oufiH&W;xDiNwxm z(zD9|IRc6|*lcy^1cI4ti6eI{09Y z)kMzfK?1ib;pOOR<;|?8TYBg-R}gt~slu%t$oOKp`vGq|ay%T=af~U(gP!$N&0|X)8q_O6B7pc!V6& zN*F@h*E43fHox7{s-I@5laJ%Pg*N!pb>-e_XUbR$Gd6viaZ-uaheSbzq4CT7h~g4l zXJ_y1AU4lp`xHvf-`h9$SW13<+vuy6CH+G?n)#*Qh(Cf?9mNJ44!;9f!#TwjH7`x4 zGD;Ch-vbwCR25BDA(#DD3 z6i#Vjj!WtpDL3bvZx`%#k6+`KGZ|K6W(ak*J0JID>XzvUS^PsbMrSp2_EN!=NzF$i zp;yB|>}O>8I(B*_T?B3Oonif15kCCAcMu$ybNS6_#MbYpA(O~SK5SI@_wTRVpa1Lx zgy}~;ysD*u>&F}Hc4f&wzDuk(%mW7KMAWWZ)@cxh_z>(&7dD&t3%z*L{wm6q;IkdS zDZKbIIwLSKnqP}7N>_s*Fs*4c?K8vVRj~1{_FH!G#)wex<%y@Wd=WV;{vg`cEJpq( zq&n%8ewX{kk_LuGGkAg?nu^#4kL!V04P#H0HDs}{<#AwX*3#DV-vsUyDd$w|}0D;1+)Wgu} zBgJHrQB!|-o4Z}ecAJOp@jKO>{r_bCvn@OnsmSE@JJ^j`|4VGO05bSW!pzO|Mw#N5 z*9YN%E&W~U8XI`TWhP%@pg~w{8^dORyJ``CsO%3N^O2+tgF+oP&KJ8Hp!y_&l8Rs5Z9ffi^zw=rCoWAB>cyp3HV0pnK{bQ6_ah4B`2hcR!OJVu^~O|Vd&#s=o@HZe5*ywDWqXlhfC zpd8%p=BV}2nwuL4K*G`#T!C4vW8*QHrimwB(5Z^h6u;Rv*1oM6~BICd(Xt7B%^K!_T)o zA7z!Uy?=Z+nXu5mFr{@J_Frq;p*NW#njl1~`d=x+} zk>C&8nHLJmeV0Pq^B~qNRqDe$5tvu{um~k4*x#mAu2+|^r$LhS)*Cz63O^F%V7D19 zr{vNlVN&)i%|A9zuK<3LBd|&{0aUaSlPdR#AbnT-QvJF0FkOGAYrYPivFgF=Kb2AOz89pSmgBypJ)QzEQLKpUFc()%o=v(9{D z3;V#KIYQ83Ql-)}tv{sN$8G55YkRf?+%0K81|;Sr&CZHc=eiPbd`Fl_CC~uZ{y%=2 zzs^LIqxr>^HB;vBH_ui2c4ZCuj~)inh%)zJyr}GLRb~@2@hum6cXoOf@>)|UJ@Unr zfY-Mf&!~Q7W`KKt;M!J}U|(<$RM?o6(lP#cR1t5vIQeF^Ra}+uDWFXbLhm57i0A<3 z4`%LhLV;^2z#hBjnTj9KX=B(Kv>P#~jB;Kr3uyYjNw0TEsrs`x$~Kn4J6fW(5ufff2~*NAIi! zP(BO`L2Qy(s{<_@FMV_W`{9FA3t4}J$aJ27J3oE}#_{lAS z;~y;yb$I4wDLO3?F98=Jk?JE&2{?WCXYSu9uz2>fPZytP|9T@3S0d9;(l1zyKs9t2 zg&|{{gHjVWTJrDw{?RoY9J`6dnWCGE5WnwrzWfv*EIu6|h7L@*P3!!e{eDh>7|y8a zVEw+0H(+zpo#+AhcN0ql^uD^W%Ec>)S|Ur?OndKR-kuO+{=2oHLhu?^`@iJ(%R&#KXCb$3`&X1c47m!ez8!xh*495B{fK8hn_IC zNv-ZV;##}1V27YbP>>_m0EQq|D)uqJm1zD5i;=F9{G!|*o!CC^=rptM#C_v1GLyF_ zc7`vONqTNF-<;*H)xbTvhlYqoif>w>I^8A$K3&&4B0>-nVs^)b(H_~Fph5u+V>Gj) zaPuAfmrN=apC_{{ap#Q?jpR(WVjk0X488J_)8^FaAMq-$G#}I1dP9{rPNL0EE|}N6 z`=F9$TFV_UtpD>28`C-%%m{B>nC(DD*j;)> zUeBFybNkFCCqA_1$i%yPFC;se4?I*d3hRshj&NdhBojEF_)ZYsE@U=zM-BI`zR}vD z>g6y!NI&)RYUg;$@n?pd;-$_Lyzo?=`A7AG8Y`o2^>3IXhvFB}| zzWLhyI#t`JPG1>v1hH5u#d(KfE_a)Ob@J8vksGiVM9n%8f=nR0I_*!Vk&>$-u=WW3 zWI3Xw#Aq_$9a~Ge{eXZZ+bCj{*-^egnjE+Z}cK}gh`$V8vb}#fn}x2QsMs#qsEf) z#3f;0bXJd#qghnmsPR`^W=;!#tJ*%YU5#4Z>#BtAhm5WYtJY4~uKP+X)Q%?@X z_pUX#;#-t$>k><6ZP^*qc8VY03Ns$v){I6*$LKO(8Ml~HFK3#m;T~MFm=ATYtkENU z^(bC8_Gm^%WQSgMDhqRYEAst5n6sIk@E#5%egRhrA*c5zFa2`2I2gSMG(Nw#{`7Vs z?+L?%3RH7%tdV2?Rx{A|ty`LE^1~w53w7?Qrn}mMLD`7cd7p6#u!xZc(`%&P*0#bJ za&Edz!okRZXOjln{1|Il^Op0%`BeuurSfv8i3cCT5RwXq@$e$Yq|`P=eCIutEcO@Y z6*77X%1BHy!i=M{qiyxj63^;uIZd9y+j&uqqyX6%;o*YM>r{NP`K757D#E?=B=}J} z7e3luNXJ9n*ID(hjd(D~m$V`o*7LE?Ot)q2WoM|tx3MV%6_H~yR1S9YBAT&HQaBDX)V~}3Z2Xhgw=V5B5!U{ zSUaUe)xVf~3Jk_Zga^7`hUGn+?1}@@w^|B)H(Z}G~Txq7yrWXl3quG?la~)f5Uu-Mv5*m%3cgWSU^iQGtBHtGa@q z?}IYQ&H+3!AX@q3A+6laZfyU?XqyEkEnBXCwXVc>_4it3e)51HOvjiPyA<^fA-CTg z2YX_mx~yJaZDT2!$EE{2?aJ_`BvaG=9}KATq+D&40+_;{RWRaAVD0C><+aF%Ic<4A zx=m%dw>bG_*>ZF7`ybAGsnlB)$Ma@p;43n@&}t&f&gT>#@e7TXjCV*|KO!us8D3Rrtq_V}X_0+xD@^g$-cu!KvPq;6%7 z4|I6hO&A~%%>Ci&U)!?Gnq@i0@m^$qwO1+BlqqqEXl&pmqARNIskA(x@qC|wDQs?S z8aw@1Z~E_dYh(Ug+Ja9tjN4_2&t0;wNanM%ConefMtg(tUU#WQ=bMOtH%$Nl!HhS2 zPQP9KA=2i=lY6|73@@5{FYA}kp~4DePxv%&eLJIH}6-b+Pb=3j?3?{lFZqxgKmd|2YI{_Q(PsrWLF1$5hO@6Elff*@Xy09)I7#%A1{N<$tMjIB4K$SkwyAe`nsC@xyc%ODI8&ut!G1~ z0`K}hq3$H+Y*aJL+D){*#Q}W%imgkTINCS9qbt_(LGdzqDqdbxE$e@80hBMGHJqAn z>A+B?=YZaR_Ntq+$8Eb68HU9Kg&_4d@yZW+XeRx47L-x;cw*4Dt^A1|LHsf-FTRFp zn^YAhomX{14_Kk&wvzhJ8`@aq8g3uQUm9 ze+S*|!%3ke#1jg*su@+;AY+)J#>z_vU2=UAa6(9OdBj^TJdpFpA?;D^vUASkl)S_snGXI6 zEsoZgpAf_1Ub2-D}U}Kb&KOf zqE=P@=6E$hpUXMar5^{}Rr~@z$Zg_)w6{LAm2bvnEn|->+{`oJMcZB06G#krk5AAn zK?-(h`kkpT>=wT0eWYr61xiYhffzXXCn<#0=zNNP#-G~#HOx0Kg$!d@I5=c=FAn}h zE9u5Peo}TNhDr*oqWgnJs?bXj=HuNvnPGPj+>G>+LIc$)q?|Gsu%l6dl?c)h5VpZ@ zh<=KpBwa(X<-UH+FlDt>hVFjVO+os16^o)qNOF+3-|$fb_*-zf0a0PO&OWb&5+B@F zx=7r0WnHJ`+#W?1-;%z+t(;ATH*N|w7Nmey@6gi^U~7`$^w zxI?0p9>{AH5A)b0D6-fI<|piKf}0()VZTH(p3GJ<5}o_`js#o9hEcsu{pXP6iqC{G z`)AmiQ&?y7748ixu5YFd#&6~GiK%}n(G$a*(vBp|R`%T0fa)oUvJvH$!B%8p+rQZn zWXe;fM#ET3Jrsrmk+avOJ@)OB@vmYUYPLjYk~^gnl(!X=ccmVx;&vne4{*e9 zY=t6z%Ri+jiRD4|o8 z34136IK)K8Rvv*+#mRC{%k*kUYhB6>ngCWu!!3pj5!sX22e5dnHnggrVfz$5**x+Zd|Pz2G@oKx&Yl)B3fGr<}dKFLvk}p?4}!Pdh@of<2@4Q8nh zeDmrr3QJ}>OHd<>FIK)VX10#;v%1mNO(+jX@fYlU#VV>2iu{$Zdpdu&$nxB8-=R$W z6u~z)ywt@ihY8?7wHqUX@gG^$U1`fkxkh=osJ&i6jCogefe92AfJ~L9@XG>oS;5?j z$?rtb*T0V-6<=;(5vCY^KeeE;?l@Z=gzfcbKr#ia<~pW%FqBkSvYImF%izbwi3tF; z_d%$Izh5`bdgfQ!C0j65qX0yt#gE?_q|+U`z$t0tC^o$87mVL-A%F$HU+MpFFVcqX zRtgeutba_S^l%@Jb`O|9aL-7v;}Q^lskc z{TD5=i1zNPR}bU@{&EBp#>bv^E$PtmJopfZ+Rw_uu#AlTC}9Ve(!8^x@mHAE?_mR5 zn9E<@B_(A`g6l|Aq;2-~`;H7>pmfMoy?|Oili!6MBQqt7us9v$Ul@?-z># z6fQb4A^NDmj)2Z&HRi1p#P`vs_F&^;Ru3o+Og~Y(6Mjzyx*+b|TzO^h>DPs@mH)AM zFBtDFiyhK1;WM+SJd^?^C+O9z6@Y^{HMdBpztl;8$tr6Y9UU77h=YUo0)lm(ar7Rg z;&Ut2tkrSSmk4bX>qVoTPtXufzSM25! zgxyWONHpXkoJ^vq279Po08d9~$_5>-$_nJt0Ie&myrFl`HJ{;$dulP(?u#RJ^~efQ;35mKSOf1wM)D6gpx%$hSD zX1M*KB!*OB^i)dZ+;PL&)DDSdQZzSWSw`mlDo*v=_x=iJm&JL9pO(Zks!(nUlnkl? zj@8OncvDpe(obFeTK?w6cX*9Fmz0NeGUG;fg-R%tgao}+4Z}W|7JCU?PZl~B&T|+> zVR>qM+B*MWHxm$BZ&(fFy|>7&o#o=;xA-(XCagm36A+8qVPg|;3HI~GJ-0MPNui8i zA{6EkeCiZHfnJbn zy3ZT&E#L_whvZ4ZC=$4+OSn-hfd-jpy7gg2+RwKZ3i|z(^$9Pbw6EL&I1PYbLy}XkF3L_I^kkD)=8JW^hlTOV^f5X=^Kd~ED{_Z zN>2~h@jf0*N2pWy{}HCnL7X9$n1J{#1pKCq|DurSPB-PWPR#%c`kxbXpAOb&U*)O8|yFrT*10(Th> z4_M@{e;%wGrSi~w4vV-xi<}^*4fzkzX*$&OK87}00JzuvhRtS!&(~W4xNj;%P$^Pi zHW3g)gS}Ad3RFQO8Ikr2IdkYUg3|b@_)FqBc)znwAzZA3bNctqeZ{5LGydE|0*&** zD>hdVURR>|f$1K0JBN*BiH%t-EHCt6fBq`Y@CrIoVd-pADEYog2vA8tVK$kqB)p%c zc4gJedxQPj-sQA?aR@mWl-mitA{L{!ec;)Nq7Zsrl!nsa1L%Ku-0%FT;Lxi!LQjA3 zz3ruKcv${!OBQVLitSe{4^??s>rzPf8ndrg7C+kCie$ zV8w%;7@pc$A`ehf$j{Ik3jor;keB|Su9Wfsb2f0BQ2LPNlsJ`QO;qlE{0)%)e|TBz z;4U(AL#G>)^WRFHG*(_>Q|jp2OO=pPuNiVmLaX>NU)PZL6|G~qNbL?DP-klW$4llU zL{sfzz8oReOY)(@ilGiJ>dW$#qjbnlx;FWkus52cN|#?C4?V9!Lb6pz)50E1E#7{k)X)!?3U zIyhVkUsips0^mx~j>6ngM6X(OrY1c{w7Jt+`3MB^!`04Kt2W4A1$K)%vC0uKbUI@( z5I`%k$+0A>J!mW!Ma@<8z2~8~j^&b{BN`h1ndoYSHPru^{6%5XJEv;pK?N z;pFD3Nr4?-@aRP5-@e_t@=hSK1OR{&iQXy2YQ$Z7hi@9rWmzAueY!HX32JN@8i0zD zGT8&}*8i@P{o&(%oU(rfhw)40V2D$$^wI&dztu zJ4@Tw;Oe;$`3JJ~9UX-j? zsdz1XGTUyY->=sg;ns8EP$>g3x2H6bfWLoVJe-ok(h@92Va2+#*|eW~YZiySGGTve z8_`}7lyh4Lm(`G_amwsnUHVEDF0wa5Fo^{$-WXiR-I0d>iKWNV+cU}wQKXC~HuOD6|Z$T0>fQKt(7hQe6F;p}8@1WzV`&1nKavZ4hmgve*L zq(pqwJ;X98r` zeyYR^UhxYm)Gz*64|L;TE7@5r8>@qN^WIePf0cp%<;?k>-dO2c7Istnxy;6RvJ#g! zFJMm(LRDO}!Dbh4w|r@u7Gr*ix*y{5N%WQ7kEE54_9;%x?M2}Hp9@fdko?f0_gqpV z@VJ<(pj;{%-J@1GgQU{MeMc0{Vhw*W(i@ZU_!Ar54YSp$dd*!oM5sBMDw#rxoFt?F zos;#rU)$t1%x-N-Q3!hyFB@+ckZj zx!{NGZ6U#A;#?1?{6G1Wm|#Pp(o2wM%yE~gg}MblxW8ELPv|E|~G zJvM!LEB;VnjS@b&CYzp0nciUyNA>$+4C|sVqRW7AW^hb=!Y>aP&;Ul{ zfoAnYY!yR{4csV3&;^Bs(*A|vLTyk=I-xcZ@i7ta)jnzpo_;I^Zn^D}tf!9qP=G8& z<+=#CQu*&oPbafwMmx1uv9Q~A4f$FLU}Lh4LF=KHSm{xT&^Fu4ad=61dmHw_uH1Iz zF5SB{F8gyDZ=N4|sEQ7y(T6};)sSVb^lS5Is|hztB7o*J0O&{KgAt$=D)P9c@x{@jW9Ixt?dn}KgZ-67#&8*A$WW2Yz#lZqa;*qx5wKN zq+FeZC=ds76mwoADvMoPh~sEmHvY*+*A+|`dun_i>w|r2aHIM5!DqAzjvHWd}euM|!ek*q<2Hv=SU(cFB-q`oPY@lnj_OfV%bG*V= z@@LZpJoNtL2X-l`ljNj8dF>pvA;gZ~irljNdyVBI%@U?J0lnR%+CdIof z>@%(|Q7?q;3p#>ed=88gW0k@b5;>)9l3j0^^Lc|;+~Qa?=hWF+;sJ_ng*oEzGYUXs z<}gMUnkYLCmCLC+kbrmRrLa-p@ft4dX~YLO-Tco>!Q(NPR-k3JhJt6W`2jU8p8tNy z@($|Y3bkG>ps*Ku)o#$xUtzx9CzNz~o9E23E1|HT!nNX^P|a2W`nc)s*2yNnHdxmm z)?zLV0l=9;{;idKl7L&(7mU{(CzCTkNl6F^cU$L$7+rMbU$|8)m5T~Gt=29rqNuIv z1-7|MWe1gO&${`lBsXYv5xF~le|LCv=~rhMc~bQw-7}-{^$KiGYeQbc`GL3IO>iu4 zHRjE~(A*jY{$kGNf--EK>=ztA+=x4BSe`;iAo#iIPKRL9nXL7}z@|~VkLvBllWAz& z8jY{i)9O)dzOqpaWyAt=aR>mHS1As~ko@`j4N!JE8#I@Qh^26tI9}#TeSV7*`F{R%G*81hZXr$zjCP$o*IYk{e7Y9KTSQ)51f0iK;^i@2L!o)C;QJy zTu;1Q`$um_(d&*-_FjF>tS)**CK+5SQ< zsO${WuhUDBz`uOEb*7wLsOewFkn^@0!2l&#CzBqn(r9N{|Uw7LJdZFFngr&NTB zx^V;5wH3fps`|CZ#$kT@ z(8)y=n;VI*6Pjg%vnetfhjl$ZKeD0hZwj0jZ#dEpy`?OGB07*(0YI^#i_>*+`28m& z0D$3e$3ru1y9L$XPP@5{L(ug3>Ejx@i0IZ$c&7W&gu@Choso)NwkLmFBRV&)+v#R|_uE0o8(10;G31=)8^} z4ymH&>ax7w*2z{u zwiyy^X(}N1e{f{b&=!dhID`%n0*7xjQ=yGfHyU5Z{=s!&`ML=hITe)K)sen=P$v%@ z@-+9|g1*xk`GedHau*dIwL4+y2wR)qPzW-8`JT}W#zZ`fA8(7 zxOd4w{fy`&E5J%WrgabLG_Q>-;?qYC=+NieLJ#FvlYgo<9I$l4p=TF#UaZtLO7{s( zt{d{wZWpxG-Q2^wbfU9;I?jt_Fq~}lUt{o-OpT$k><8w89{tq$bS3|tF%<$ZiNQ7-KANE3 zpox_m7oW0vO-x@qnwmn1!wV$>{ZwfTQ}-L@eG@6sB4ahQ=jbGh0)Dj?(PO5*>j#Lb ze;&U(6+2-*%tM*`Y&JKDGgneZW^Ib~b<4T@W8Zr}$u8YKJTGxye_J854Q0OK8x9iD zNgfthG^(N)P)-8Wzf(LTiZ7!ZPMF3oA9u1T05|F;G1=5@2 zk_XRqG6GL2oiqu3jPG1czBA&kLrtszybWZBug_Qz+y(_h7;j>i%5FU(+R#K>w~2iF z$ZBHT&`+j@x3$q5u;d`Q*M z87~_kdrKSIuTcB^914f8BcxGPTdUlzQX#ybYYY;bp3O4%&gf9xq?GQ-!Vircd}HgUdf zJcYabCk6V2tV1p_Dhl6ZJU%3dZz-w`;O1F=pi%}ZEVSwVwR>FTU>8(ZETzqkbiz*C zr`wNK4ix%b-G6(qk%hG%{{*J%1~Wm#R?U^ix09DdN+9C(--eSL0Yr;Y_%_5J7S`d< za!%)a0@FwC)6h3wscsa!^7S>5ISa7g@bg7}it{@+p9eCto zJ^{^xv9+t0p1o8-zeJv=#&1}J2=>t;D68AbHeM*Uhb5Bgl+f5jzi4Jk(!0 zB1UzSJEq^c=1CJwixH-<&+NP?@dZ{;_C9HDO!RQ!89Gloiv7p5wmR4FmE&_!!-Xr~ zKV5<9GG{l1FIqinC5^k10-q9FUL-*Pyoc8scCLkEu8o_=Zm4z)Jo=(Eavd_it-H(p zX4D2QzN2NXFJ@g2-FEx=yX*=y9NmmQ(v)yRxL=ILhtB!u$G)Mf`~5U3(;-mV3>_bwX$fV1td4MP*mki zn-Nq8nK4UHp0W6KE-OUf+<~&rTa_ZrIM>lHg{e zf^g9*??+}aodCV09m+`CB(P6e#7PF>h20sMw|;005vu|FG~ss-`m^#jh880B>yMSn zlL7dAwGyJ;0^^f8rP1`W%f!%M+zB=%>Kx#*U|_i^q#~p=@u25WSC?a6qek!qnLN9UiP41U&!GzvYTzHuy4thvvI)fhOkIEO=s)ksSd zevzCqHdzMuTKxp>ta(5o@)v|#1bg%5)ibuLJkzO%&{RwNWp?k3n`Z!evreW@FQCo# z`48kNkI5$$6HoqQ(;Yuz#CmO#DHs|Ybc04x@@%>yVboc_6NM-B(`{o?#RfmjAYvR@OSJ_n|J0!to~BY;3t=Tikh z_anoaUY_t*%4M5oO7N8CB^r8g=JaRC=?cS&7RsFRG}j3~V3e|te|^fp)Z3?mVMh(y zT!+wVem_kcN^%|&^9!N)cGeJL1i()EKP6MeR-8*@`Fb|+ln9!T69o10=0OoI^0RIn zKLSe-j~UO=HI5j3Ah&#W-VO+pg%k5&w}c}7<`lxaPLRvM{#>>yMl zO^@IDi76H@7_x?SzI-oKPKpE8o~#`ow_OyC&0*yBzTx9gidm5tmh_%{R#LzJ?tZfD zl|%Okk$6JzTT-aV!vRnD;?2&G^wpiw>sltPu~1au#bo+Y~!LoeB3ioeHsOp}=uCYP^;9tpcX;yeSyY-gQ zzSysSso*C-=&!CLxo3gST=J9Y+lQ9}4yYST2EnrL{5RTc*1K!qLL@gy#`k4l^`C#X-Iq0KS6gnDereAh>ItI@-#9v~zM+cHYCHEp1^2!SZg^EWR zk}~;n;TV3Be-!Dlb{Bcby#1kjh3vxo!cS|lb#QybU$n=JLiP-{Csl0zEr%GIKDICt z9pEXJjM;1uC~Ib?fe3H@y_QpWRFQMTTqpR_#~f-qWkzU{wSMyKLSJ)JikQd+krEGt zBO^`eNwK*DRii^sO`i&8p@L3(mc_uSe;DZS&`iSkBg@=0DpG1TEs~B6>KDi<9Jyd{ zvDp6mrGVLwp+7Kza?oYQ6f4vyQf%L0v$O75kYQ z0|kp!N(%Lx!er43lu)}Uw#P8Vt{FJzXu`0?Kxgqr~*#?z)Zg|iR#7Vbz~Ab}dx zPy57N>`oP+fU7H`AwoR;U;g?Yxge|oDjJ39eTPc#G_x`O;I=TOAmhE8N%+Tk!U8S` zeG60XONTYQ(6K}n9OL4TT!_6`EX6OqtKekQ^pq5m`$`N$;0+@yEFH!7L3tNpW!{t7 zNj0}9n185sP`*FTO(1ql%Hrz!IMJ6j@p0bG~Xd^G1N2v;3>U(+n<+u z_xTCWm{b^HYOX5{g)vRj!w%0neV=~+vzaGw%ZTGxi=R#-?c&6tHlJOPVwocJ8z{N#z*CqL1SY$`y&-mYoC5Yo3Zux@E@sk@iTeM zl;wO}4Iv%fXVh>ZR6G-Asl1K>X{ka*oy7}0+qcC<&iHdw^QtgNOKr0ekQv8cw+E+f zoFKE;GsCB@KMkJoH|d4u7Jg-DBnR0xvfbyPl1?h-5F`b>wUa2&fpW6LmWca`PG#&5 zpSnOs_ry%?lk8SNz14g;OuG3oKPD)(zv;^ow&Zw`x3D?}9#s$z#kx?VMeSj7$E9 zI<2Kh@jE2~$JdblDEhv#S!)$XpxyZ$d->}hyWfh_#m!yC4-l#L>!MN4YqyZoz1JfC zpfn1*Dpw2!^!ZM;$}uU@c?N0w(9ROyGRdlLI&eOgpoh|s$K_2-UIoELzra;JLbvk8 z+1_%qfy%}F#S0cy&rT&!t}1d{a%P!LV)Wh6L}tas3-1x?;BmTM`{dfyN)yQy{YjN#oDg2g`~&HXw%>b1d<(jXonigsNh1EG?NSX>(O%{KLC z_bPIcQJL*+$1N+BFgKpA=`v{4O?Z*)@l|ld=mmud5Q?2wM$-j;#Qtl?TCB2ukEz2& zVAFjxeIH9A&va=*gz?96i`&DVzCu48oO76>5WqkYXL@cmDFnla7HeuNMr^5f!4L2Bp7TPlvav1qL0J5 zmOp*!zm$4ls~UdFjw3qV>zxmmrdY=YFFCn8?>vXFBV3Y|RZ^K3f0ps0iatKgw z^9#>^_=Yvpk}njCdaVY=PtJ6MeI5_;NdmPT?@LEBn%`kX^zpmDOA)6lSFXw0^nLk&gTQaOTH;}Bz{j#-2Rr3Jxzv%X z!EVql>k1dLcc*s}fCq5qyIKl%8PdnMS*rBswAx_4pZ}n7c)%f3k)WSH-&*QK_{2-^ z_psN2kxiwC0|PEaGCKK@yGb%$|`M10O@@ugN{^Co@cCgG1lmxb}p-7#^yztl>rus3wq-CU$9 z*4OpW)ttJn+SYpdUXN;-jPa+J&bRJ}7XN5jko#jt9aWII+$bg!>UXc&BwD`s?fOfs zUMQic#hWM#OJ!|_iVhx6k{0;tQM?6WhP@afXm@+NWu z00vuwl$BZifeN=&e93==^jJhewXa`9u~qL&jDLG)v``vgIQaXik4j$agQv6d@OHC1 zRUZK$)UzrcY4*@o@E7kgAC#`#KW{BPqTOhu8!o!9kIt#(R{zbr5+J0rmy>ftQJGz8 zTaiBlA+d+6XNo_vr+{9{LA#MuiRL@wC>`o|kBYhOvGpOhsvmv$A)3(Td7C0UtWX(U zai}Axp@UBOLlqQg7rC-Iv)yp5pH5iZ$XViTtUfs6{-YtgCt@=L@B5+Pv1nfgL($yv ztl5G2B39&CYiCT>a;qatt6fLK81$;~1fD|kqd2^BQsf!zll|2pXEbcR$%%g$B?hBi*`%IFRjxQeS^crt#w1bfc=#`ecnc{ zJ=n1{4sV}rZ8#j4e`brt2XX=osnpfC7knl6qSa$xUG|v~Y;_s^an&(49E`VS>L+V3 zxW`HiIHEKfIFBtr%2y2WJ!pczfYEW_*5y=u8xLT@ZKHT*4T9b<{V3HW9i)oc3iGaj zzT@yrDZz8r&7rvjZ~AB;N9+ zrslk!`+e1p)(tV^?_&ZAt4mKQt`^L!591t`)y0gC8XF~oi$==+;1efkRi&$e$~7+edf7|YnaU%Y9-Cf86nv0t=VnT3pFqfumLFJ4#8zmfeee<=8aP(*FLV`u=r z%}+r>?xh55)PLV?<$55GE{FknDw)yK9d`sKY@ii+5b|c(j+@Xf%)CV0P$l;Dv}HKU z$#)K+i;UjYlIC6=H}uI;;=|$5OQ8^NYv*{VI zWHIPQR~7>rYQ8+XopIM~=c(5ZvUl`yjtk>Qi=o^uRpf6oO)N%`@SSaNFZW9ZP z8hd`6PSpU9sMfld;2g)3H5?V!%s80m&|N3nkV&w3`ljRe6ZxPozbQ4ar~P{UNCbJw z&%!@iw5Ls{xW(a|MZaua0J5U!%T&K$gu(M4l12RS(|SG=rJ`jIVl_~=)L}z+pE+r~ zw9`Ts>zmZQN?O&sYtU0#CcakJs~}y9^8V~FzDu@l?jNv@BwjU;?WiS8ap7G{JAIXJ zEcjuX@MwG?+HKg_R{bbt$XYMk{Va zKO6za^E|DTG#T7#yOa6DM}d@fEqSd)M17eu5@|#MfmFJdjEj|RH!oe@w$Y^r0f9hR zo1%CPls?9tyk+fyM2{5PRSo-^^KvZ}Vv8xUelB!VyH~b6g+$vp$ClllLb^zovk%JI z%u0@*Mk~Ibz>K&t84yT!c|>y`OWV9#^;hS$yjF1#~uK*WoX5>9{Kz*UR`dX=*B1_>N19Y{wWt{({FzCj1VW!w+@q(H)$} zR=#Is>+si=Bw|t<>(kW`)sx-ToR*so^lwF$oY%K5wNDlQu#i zT|Pk}UcKk7x9CApbg$M-4AW(WU4~EJXNNcrDpu(vCKy>qT%!wnCPtYI?{bAXYhu4; zP~kQ2QujYPhCY7@!R2b;?u{7a7Vl2{aVGAATV{#~yyV z?6CJj2x-sLs`C&4S-ehSVaJvS&C4FwC2~zV*JF|!VBW-*2~gfkO${xsH!7kz0iU5r zyWvs3q@`%wV~!icf-w@eZ_G`L`wMM$-V>)^w{i$;iE1xnlQVdjtG!1mvr2RDe}Kic z7+K2qguPxpMfJsWpUTUev{jah=kYA`=`%j>3Zh+He9N(FZ5$UErNgN$$j`sko*T9J zP_wZ~`ha3Ru(_fROX&L1(UGq+yP+AV>o_{{RpNvhRBjA7vqEz%_c)78kb+AnGR2*np5_bxMC|Mn~LrO8m(! z#&X8o_=Q>8XvK@l{STwX*U$K8C<(xup9tuL+kf0lZba7*uqbn{xdIJGjfS(Yzr2#I z{zyA4$70nO1unazOK{YE&YRr8O@zSn8tj8*S0Z6aA(k+TT>Hih4+VnLE8OYomUq2A zGs9U!MwnnfFSXB?p&atRWtY9~$j?JQwXO-t5R*k~BOBjj$3Wc-LeCvbtN4S!xx+j- zoj36N0?aAhfaQ75h`>|d^D1p8VTP+cUu7?+Rbb=F9<5MFw}t7<7kwSao}xtxZm=&G z>5;jCf#~9s0EXea95?Wu3oElji~00!zd3>$k9ib(wy4bs@>3f*)t&1p-d~nG^=j$n zBe8-oH*;_zL*w~}jSsY-LQ3mGP&qU}TWglTBf|p~G!*PIePszr#01(_Poj=lao-96R%ZITs?lp%({hRv)f?M@iMI-cXox{Z1_Rp7TXsRc_^AYXj4gIb#-BuET`i4G?3TmMs8h zHBXmCHgw=g;J6Iqs`>ryKyki$yo`e+u`lCkKZ!V=3!~J;zc86J8E9CQU?cu;Fpp z63*7rw{+(kN$7pCRH$OJ9I7}lx}=7wRoy|;d_VkS?h^iF zsdkz4mg&J1eLwmiGiO&I_i8Mfcg=oE*)1gKAyKC{Zry(Wln8B|p5GMD$~P(e;A8_a zW#USv$@fu=j*9ap7W%ZlHWmCkc43)ui1mE7zF*#@?BujkziWFss2S#=%dOdc$f@eR zEmf#su9lc4Hsdl{fKGLiy;bU@46@+^o1DJcrWP`V=XjhYB^K(nTRGqu@b2n!eeM%9 zu$SIKw?9ZKy=ZYJ^Ux*g-yHPGQnNp_zH=rG@epuYJa*sjuPERFFCcwv>F!-}1eD$3 zG18BSJ@_F7xS>lDj1dcJ_Vbq$p9I!lY?*l2@S%kwG9|AY~_XOx};jFnFa!s9Eir1R_!>A)pmT0wu_kx z0u3Z=Vs##TCs}XChp+YcFkZ?OpdNl6@t5o!hZzbyTgtbX+gRg|&kb7s5CP0+Myl(I7`y?B|elyL5j3)WJ&7i|R80Rlz8oS_(S-Cos!>yI5 z$h4Qd@3j|oK>H6Lf9qRvsioECSwK%Ld|FWr7HB&Xv?(T<{Jx}C3VT>vB`lZ}dal5M z{W{?$=^IKng+XT~UMEm~_JTD`N5nn8_~Ym?*mwqDOJWBuMXMc$1z{ThSF}oEtmhTj zD@o^9Gml&`GF+0cfAOqfDx&Ejrg2}|e9TgDtJP|-f(sNF-3S*O6)%z6U|;6J)xxw) zACJHamw|_=ev@t`uX($%n%f6&D)yxhsk=2gER&4*k{(&aNZ}n_R zlR<-%A?&8;rPi3LJR`%lz#K>D_H%8nw&UTQ-+k$Ut1d$Fb{*_M3cz;ZWL99p*Q zq=zJtdC5VJq%HLepNesrP>d#AuWa7SFWx<)rS{U>%j zGIb_*20P$u_ishNRlX8*;oY9?%3DF#>blEWFD!0L$ujo!-jy{qp=Aa0ri(mchro2X zPekKn@K8I)5$V8(^83q-e>&|B?kz-rE+;4hA+9kEV?T1rJ%bT~PmQ7QzH)P%_7Lio5Xf?Gw-uw)8ip^}X zU0*Fj_6$Fkv0?_o3dJNsr+0|JkhWX`@CWOR21w|i%^USKT5`Ep;L*N+iBCDmMpNnG z*{LP9F18fy*=i=AS$n)S1OirX3JR;?2iW`0ixqe{#;*hTg-6|Jj&=ooiNC zeHi<%-xnXwc#?5n>y2Y@H}FspV(%O!^W$yWa%W!Q6aOwF{_s*1Lbz(?w4|K@%`o)^|goH^*-2~%fM|sLBG4d|E3-vvgI!uEV z5=EDkkQ+rQdvSReBgH5iHWRX2zYid9ZP}H9iRw~d5jOm`(zs60lRbwh5DfamB%!N5FRktOn0bc5 zVa7z*#6T{B@_Uz-yj~Qu6UiMR@~6?O@q8uHE&jgetDAFhZncPwuc|uR^%V!IdjSg^ zaXFv^7==?EHHDvwEYhYQhlG8C6+uTH6>-!p)ccLI(6><){{+=U+1)L#u>`_$l*V|k3E@~jSeN)CJAT3mJHTHfNvKOYmHcb+1n*K7C zb(+|GHnFjU{OLX4W)p8QYzSixwZ1T1FJ@siX2BEB+iB2lLex7VD0~VL^bcZP?H2iR zxjOzWMUW;Ahq~NhVl!znfy9M#b3(vs?2LmamCcvAsjB!| z8epg$oRnAJoL7(EWOh$`^pUyo?6Ei;mdTMIb%U^>@>}RJ}l={z)m+sJA>7tfQz&%u#ZJDH4z=HQCHnS!+>uF(0qx{wg zP*!$cJvpzQrTS@R-`Pu?C$+H7Dww}|@$n9)z||0Q??ES-vzx%5W)GIDf5YiO%s%j? zf)Wi#;Oi(R6VZ2{X3aj}jVel^5fmP4kwc}cpdKOQ*FVKK-HdQs&ieF_I^0eKsQ`jc z&y;tjC|upnIpdk_(6ZB>2=v)-rqv zKiZ(No*7~|5oQJlXNEmN33W^Wzz}&H|Na6KNFSvpIR8H+4vgmk3;%;?fwy=d%YP9t zqdJx4KM0tEoyzhbN JQx4ci{SSGb(sKX+ literal 0 HcmV?d00001 diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md new file mode 100644 index 000000000..2fa7a5493 --- /dev/null +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -0,0 +1,355 @@ +# NGNIX Ingress Controller Canary Deployments + +This guide shows you how to use the NGINX ingress controller and Flagger to automate canary deployments and A/B testing. + +![Flagger NGINX Ingress Controller](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-nginx-overview.png) + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer and NGINX ingress **0.24** or newer. + +Install NGINX with Helm: + +```bash +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true +``` + +Install Flagger and the Prometheus add-on in the same namespace as NGINX: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger flagger/flagger \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx +``` + +Optionally you can enable Slack notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace ingress-nginx \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Bootstrap + +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployments, ClusterIP services and canary ingress). +These objects expose the application outside the cluster and drive the canary analysis and promotion. + +Create a test namespace: + +```bash +kubectl create ns test +``` + +Create a deployment and a horizontal pod autoscaler: + +```bash +kubectl apply -f ${REPO}/artifacts/nginx/deployment.yaml +kubectl apply -f ${REPO}/artifacts/nginx/hpa.yaml +``` + +Deploy the load testing service to generate traffic during the canary analysis: + +```bash +helm upgrade -i flagger-loadtester flagger/loadtester \ +--namespace=test +``` + +Create an ingress definition (replace `app.exmaple.com` with your own domain): + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.exmaple.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 +``` + +Save the above resource as podinfo-ingress.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-ingress.yaml +``` + +Create a canary custom resource (replace `app.exmaple.com` with your own domain): + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # ingress reference + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 10s + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # NGINX Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + # load testing (optional) + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://app.example.com/" +``` + +Save the above resource as podinfo-canary.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary.yaml +``` + +After a couple of seconds Flagger will create the canary objects: + +```bash +# applied +deployment.apps/podinfo +horizontalpodautoscaler.autoscaling/podinfo +ingresses.extensions/podinfo +canary.flagger.app/podinfo + +# generated +deployment.apps/podinfo-primary +horizontalpodautoscaler.autoscaling/podinfo-primary +service/podinfo +service/podinfo-canary +service/podinfo-primary +ingresses.extensions/podinfo-canary +``` + +### Automated canary promotion + +Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators +like HTTP requests success rate, requests average duration and pod health. +Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack. + +![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png) + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.1 +``` + +Flagger detects that the deployment revision changed and starts a new rollout: + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 0 + Phase: Succeeded +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Scaling up podinfo.test + Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Normal Synced 3m flagger Advance podinfo.test canary weight 5 + Normal Synced 3m flagger Advance podinfo.test canary weight 10 + Normal Synced 3m flagger Advance podinfo.test canary weight 15 + Normal Synced 2m flagger Advance podinfo.test canary weight 20 + Normal Synced 2m flagger Advance podinfo.test canary weight 25 + Normal Synced 1m flagger Advance podinfo.test canary weight 30 + Normal Synced 1m flagger Advance podinfo.test canary weight 35 + Normal Synced 55s flagger Advance podinfo.test canary weight 40 + Normal Synced 45s flagger Advance podinfo.test canary weight 45 + Normal Synced 35s flagger Advance podinfo.test canary weight 50 + Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test + Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test +``` + +**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. + +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-05-06T14:05:07Z +prod frontend Succeeded 0 2019-05-05T16:15:07Z +prod backend Failed 0 2019-05-04T17:05:07Z +``` + +### Automated rollback + +During the canary analysis you can generate HTTP 500 errors to test if Flagger pauses and rolls back the faulted version. + +Trigger another canary deployment: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.2 +``` + +Generate HTTP 500 errors: + +```bash +watch curl http://app.exmaple.com/status/500 +``` + +When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, +the canary is scaled to zero and the rollout is marked as failed. + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 10 + Phase: Failed +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger Starting canary deployment for podinfo.test + Normal Synced 3m flagger Advance podinfo.test canary weight 5 + Normal Synced 3m flagger Advance podinfo.test canary weight 10 + Normal Synced 3m flagger Advance podinfo.test canary weight 15 + Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 55.06% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 47.00% < 99% + Normal Synced 2m flagger (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99% + Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10 + Warning Synced 1m flagger Canary failed! Scaling down podinfo.test +``` + +### A/B Testing + +Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. +In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. +This is particularly useful for frontend applications that require session affinity. + +![Flagger A/B Testing Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-abtest-steps.png) + +Edit the canary analysis, remove the max/step weight and add the match conditions and iterations: + +```yaml + canaryAnalysis: + interval: 1m + threshold: 10 + iterations: 10 + match: + # curl -H 'X-Canary: insider' http://app.example.com + - headers: + x-canary: + exact: "insider" + # curl -b 'canary=always' http://app.example.com + - headers: + cookie: + exact: "canary" + metrics: + - name: request-success-rate + threshold: 99 + interval: 1m + webhooks: + - name: load-test + url: http://localhost:8888/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 -H 'Cookie: canary=always' http://app.example.com/" + logCmdOutput: "true" +``` + +The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or +those that call the service using the `X-Canary: always` header. + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.5.0 +``` + +Flagger detects that the deployment revision changed and starts the A/B testing: + +```text +kubectl -n test describe canary/podinfo + +Status: + Failed Checks: 0 + Phase: Succeeded +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Scaling up podinfo.test + Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Normal Synced 3m flagger Advance podinfo.test canary iteration 1/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 2/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 3/10 + Normal Synced 2m flagger Advance podinfo.test canary iteration 4/10 + Normal Synced 2m flagger Advance podinfo.test canary iteration 5/10 + Normal Synced 1m flagger Advance podinfo.test canary iteration 6/10 + Normal Synced 1m flagger Advance podinfo.test canary iteration 7/10 + Normal Synced 55s flagger Advance podinfo.test canary iteration 8/10 + Normal Synced 45s flagger Advance podinfo.test canary iteration 9/10 + Normal Synced 35s flagger Advance podinfo.test canary iteration 10/10 + Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test + Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test +``` +