Implement NGINX ingress header regex matching

docs/gitbook/SUMMARY.md

@@ -29,7 +29,7 @@
 * [Contour Canary Deployments](tutorials/contour-progressive-delivery.md)
 * [Blue/Green Deployments](tutorials/kubernetes-blue-green.md)
 * [Crossover Canary Deployments](tutorials/crossover-progressive-delivery.md)
-* [SMI Istio Canary Deployments](tutorials/flagger-smi-istio.md)
+* [Canary analysis with Prometheus Operator](tutorials/prometheus-operator.md)
 * [Canaries with Helm charts and GitOps](tutorials/canary-helm-gitops.md)
 * [Zero downtime deployments](tutorials/zero-downtime-deployments.md)
 

docs/gitbook/tutorials/prometheus-operator.md

@@ -1,78 +1,69 @@
-# Flagger with Prometheus Operator
+# Canary analysis with Prometheus Operator
 
-This guide will show you how to use Flagger and Prometheus Operator. 
-This guide will handle only Blue/Green Deployment with podinfo application
+This guide show you how to use Prometheus Operator for canary analysis.
 
 ## Prerequisites
 
-Flagger and Prometheus Operator requires a Kubernetes cluster **v1.11** or newer
-
-Install Prometheus-Operator with Helm v3:
+Install Prometheus Operator with Helm v3:
 
 ```bash
 helm repo add stable https://kubernetes-charts.storage.googleapis.com
-helm repo update
+
 kubectl create ns monitoring
 helm upgrade -i prometheus stable/prometheus-operator \
 --namespace monitoring \
 --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
 --set fullnameOverride=prometheus
 ```
 
-The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false` option allows Prometheus-Operator to watch serviceMonitor outside of his namespace.
+The `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false`
+option allows Prometheus operator to watch serviceMonitors outside of his namespace.
 
-You can also set `prometheus.service.type=nodePort` if you want to have access the Prometheus UI
-
-Install Flagger with Helm v3:
+Install Flagger by setting the metrics server to Prometheus:
 
 ```bash
 helm repo add flagger https://flagger.app
-helm repo update
-kubectl create ns flagger
+
+kubectl create ns flagger-system
 helm upgrade -i flagger flagger/flagger \
---namespace flagger \
+--namespace flagger-system \
 --set metricsServer=http://prometheus-prometheus.monitoring:9090 \
 --set meshProvider=kubernetes
 ```
 
-The `meshProvider` option can be changed to your value, if you want to do something else than Blue/Green Deployment
-
-Install Flagger Loadtester with Helm v3:
+Install Flagger's tester:
 
 ```bash
-helm repo add flagger https://flagger.app
-helm repo update
-kubectl create ns flagger
 helm upgrade -i loadtester flagger/loadtester \
---namespace flagger
+--namespace flagger-system
 ```
 
-Install podinfo with Helm v3:
+Install podinfo demo app:
 
 ```bash
-helm repo add sp https://stefanprodan.github.io/podinfo
-helm repo update
+helm repo add podinfo https://stefanprodan.github.io/podinfo
+
 kubectl create ns test
-helm upgrade -i podinfo sp/podinfo \
---namespace test
+helm upgrade -i podinfo podinfo/podinfo \
+--namespace test \
+--set service.enabled=false
 ```
 
-## Setting ServiceMonitor
+## Service monitors
 
-Prometheus Operator is using mostly serviceMonitor instead of annotations. 
-In order to catch metrics for primary and canary service, you will need to create 2 serviceMonitors :
+The demo app is instrumented with Prometheus so you can create service monitors to scrape podinfo's metrics endpoint:
 
 ```yaml
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
- name: podinfo
+ name: podinfo-primary
  namespace: test
 spec:
  endpoints:
  - path: /metrics
  port: http
- interval: 15s
+ interval: 5s
  selector:
  matchLabels:
  app: podinfo
@@ -88,31 +79,31 @@ spec:
  endpoints:
  - path: /metrics
  port: http
- interval: 15s
+ interval: 5s
  selector:
  matchLabels:
  app: podinfo-canary
 ```
 
-We are setting `interval: 15s` to have a more aggressive scraping
-If you do not define it, you must to use a longer interval in the Canary object
+We are setting `interval: 5s` to have a more aggressive scraping.
+If you do not define it, you must to use a longer interval in the Canary object.
 
-## Setting Custom metrics
+## Metric templates
 
-Prometheus Operator is relabeling for every serviceMonitor, you can create custom metrics to you own filter.
+Create a metric template to measure the HTTP requests error rate:
 
 ```yaml
 apiVersion: flagger.app/v1beta1
 kind: MetricTemplate
 metadata:
- name: request-success-rate
+ name: error-rate
  namespace: test
 spec:
  provider:
  address: http://prometheus-prometheus.monitoring:9090
  type: prometheus
  query: |
- rate(
+ 100 - rate(
  http_requests_total{
  namespace="{{ namespace }}",
  job="{{ target }}-canary",
@@ -127,9 +118,34 @@ spec:
  ) * 100
 ```
 
-You can also use `pod="{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"` instead of `job={{ target }}-canary`, if you want.
+Amd a metric template to measure the HTTP requests average duration:
 
-## Creating Canary
+```yaml
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+ name: latency
+ namespace: test
+spec:
+ provider:
+ address: http://prometheus-prometheus.monitoring:9090
+ type: prometheus
+ query: |
+ histogram_quantile(0.99,
+ sum(
+ rate(
+ http_request_duration_seconds_bucket{
+ namespace="{{ namespace }}",
+ job="{{ target }}-canary"
+ }[{{ interval }}]
+ )
+ ) by (le)
+ )
+```
+
+## Canary analysis
+
+Using the metrics template you can configure the canary analysis with HTTP error rate and latency checks:
 
 ```yaml
 apiVersion: flagger.app/v1beta1
@@ -145,40 +161,37 @@ spec:
  name: podinfo
  progressDeadlineSeconds: 60
  service:
- port: 9898
- portDiscovery: true
+ port: 80
+ targetPort: http
+ name: podinfo
  analysis:
  interval: 30s
  iterations: 10
  threshold: 2
  metrics:
- - name: http-success-rate
+ - name: error-rate
+ templateRef:
+ name: error-rate
+ thresholdRange:
+ max: 1
+ interval: 30s
+ - name: latency
  templateRef:
- name: request-success-rate
- namespace: test
+ name: latency
  thresholdRange:
- min: 99
- interval: 1m
+ max: 0.5
+ interval: 30s
  webhooks:
- - name: smoke-test
- type: pre-rollout
- url: "http://loadtester.flagger/"
- timeout: 15s
- metadata:
- type: bash
- cmd: "curl -sd 'anon' http://podinfo-canary.test:9898/token | grep token"
  - name: load-test
  type: rollout
- url: "http://loadtester.flagger/"
+ url: "http://loadtester.flagger-system/"
  timeout: 5s
  metadata:
  type: cmd
- cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898"
+ cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test/"
 ```
 
-## Test the canary
-
-Execute `kubectl -n test set image deployment/podinfo podinfo=stefanprodan/podinfo:3.1.0` to see if everything works
-
-
+Based on the above specification, Flagger creates the primary and canary Kubernetes ClusterIP service. 
 
+During the canary analysis, Prometheus will scrape the canary service and Flagger will use the HTTP error rate and 
+latency queries to determine if the release should be promoted or rolled back.

pkg/router/ingress.go

@@ -159,19 +159,20 @@ func (i *IngressRouter) SetRoutes(
 
  // A/B testing
  if len(canary.GetAnalysis().Match) > 0 {
- var cookie, header, headerValue string
+ var cookie, header, headerValue, headerRegex string
  for _, m := range canary.GetAnalysis().Match {
  for k, v := range m.Headers {
  if k == "cookie" {
  cookie = v.Exact
  } else {
  header = k
+ headerRegex = v.Regex
  headerValue = v.Exact
  }
  }
  }
 
- iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie)
+ iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, headerRegex, cookie)
  } else {
  // canary
  iClone.Annotations[i.GetAnnotationWithPrefix("canary-weight")] = fmt.Sprintf("%v", canaryWeight)
@@ -202,13 +203,12 @@ func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[strin
  }
 
  res[i.GetAnnotationWithPrefix("canary")] = "false"
- res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
  return res
 }
 
 func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string,
- header string, headerValue string, cookie string) map[string]string {
+ header string, headerValue string, headerRegex string, cookie string) map[string]string {
  res := make(map[string]string)
  for k, v := range annotations {
  if !strings.Contains(v, i.GetAnnotationWithPrefix("canary")) {
@@ -217,7 +217,6 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string,
  }
 
  res[i.GetAnnotationWithPrefix("canary")] = "true"
- res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
  if cookie != "" {
  res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie
@@ -231,6 +230,10 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string,
  res[i.GetAnnotationWithPrefix("canary-by-header-value")] = headerValue
  }
 
+ if headerRegex != "" {
+ res[i.GetAnnotationWithPrefix("canary-by-header-pattern")] = headerRegex
+ }
+
  return res
 }