[stable/redis-ha] Added HAProxy to support exposed Redis environments. (

#15305) * Added HAProxy Templating / Support for LB Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Updated some nodeSelector values Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Added better message to config-init script Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Cleanup / Added to README and reverted defaultS Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Bump chart version Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Added HAProxy Templating / Support for LB Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Updated some nodeSelector values Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Added better message to config-init script Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Cleanup / Added to README and reverted defaultS Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Bump chart version Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Removed previous attempt at separate chart Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Removed trailing spaces Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Fixed selectors. Working perfectly! Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Fixed deployment spec Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Fixing review comments. Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Adding all AUTH in HAProxy conf. Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Added tcp-check expect to auth check. Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Updating scope within range. Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Changed var to in range Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Added Helm Test/CI Tests, updated existing test Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Fixing linting issues Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Fixing review comments Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Adding livenessprobe. Removing bad test. Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Merging with master Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Upgraded HAProxy version. Fix Segfaulting Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Upgrading chart version Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Adding myself to redis-ha reviewers/approvers Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Removed myself from reviews. Need to be part of helm org. Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com> * Removing myself from contrib list until helm approved Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com>
helm · Aug 28, 2019 · c6d7f8b · c6d7f8b
1 parent 903abdf
commit c6d7f8b
Show file tree

Hide file tree

Showing 11 changed files with 309 additions and 49 deletions.
diff --git a/stable/redis-ha/Chart.yaml b/stable/redis-ha/Chart.yaml
@@ -6,7 +6,7 @@ keywords:
 - redis
 - keyvalue
 - database
-version: 3.7.1
+version: 3.7.2
 appVersion: 5.0.5
 description: Highly available Kubernetes implementation of Redis
 icon: https://upload.wikimedia.org/wikipedia/en/thumb/6/6b/Redis_Logo.svg/1200px-Redis_Logo.svg.png

diff --git a/stable/redis-ha/OWNERS b/stable/redis-ha/OWNERS
@@ -1,4 +1,4 @@
 approvers:
 - ssalaues
 reviewers:
-- ssalaues
+- ssalaues
diff --git a/stable/redis-ha/README.md b/stable/redis-ha/README.md
diff --git a/stable/redis-ha/ci/haproxy-enabled-values.yaml b/stable/redis-ha/ci/haproxy-enabled-values.yaml
@@ -0,0 +1,8 @@
+---
+## Enable HAProxy to manage Load Balancing
+haproxy:
+  enabled: true
+  serviceAccount:
+    create: true
+  exporter:
+    enabled: false
diff --git a/stable/redis-ha/templates/redis-ha-configmap.yaml b/stable/redis-ha/templates/redis-ha-configmap.yaml
@@ -50,7 +50,7 @@ data:
     set -eu
 
     sentinel_update() {
-        echo "Updating sentinel config"
+        echo "Updating sentinel config with master $MASTER"
         eval MY_SENTINEL_ID="\${SENTINEL_ID_$INDEX}"
         sed -i "1s/^/sentinel myid $MY_SENTINEL_ID\\n/" "$SENTINEL_CONF"
         sed -i "2s/^/sentinel monitor $MASTER_GROUP $1 $REDIS_PORT $QUORUM \\n/" "$SENTINEL_CONF"
@@ -135,3 +135,79 @@ data:
     fi
 
     echo "Ready..."
+{{ if .Values.haproxy.enabled }}
+  haproxy.cfg: |-
+{{- if .Values.haproxy.customConfig }}
+{{ .Values.haproxy.customConfig | indent 4}}
+{{- else }}
+    defaults REDIS
+      mode tcp
+      timeout connect  4s
+      timeout server  30s
+      timeout client  30s
+
+    listen health_check_http_url
+      bind :8888
+      mode http
+      monitor-uri /healthz
+      option      dontlognull
+
+    {{- $root := . }}
+    {{- $fullName := include "redis-ha.fullname" . }}
+    {{- $replicas := int .Values.replicas }}
+    {{- range $i := until $replicas }}
+    # Check Sentinel and whether they are nominated master
+    backend check_if_redis_is_master_{{ $i }}
+      mode tcp
+      option tcp-check
+      tcp-check connect
+      {{- if $root.auth }}
+      tcp-check send AUTH\ {{ $root.redisPassword }}\r\n
+      tcp-check expect string +OK
+      {{- end }}
+      tcp-check send PING\r\n
+      tcp-check expect string +PONG
+      tcp-check send SENTINEL\ get-master-addr-by-name\ mymaster\r\n
+      tcp-check expect string REPLACE_ANNOUNCE{{ $i }}
+      tcp-check send QUIT\r\n
+      tcp-check expect string +OK
+      {{- range $i := until $replicas }}
+      server R{{ $i }} {{ $fullName }}-announce-{{ $i }}:26379 check inter 1s
+      {{- end }}
+    {{- end }}
+
+    # decide redis backend to use
+    frontend ft_redis
+      bind *:6379
+      use_backend bk_redis
+
+    # Check all redis servers to see if they think they are master
+    backend bk_redis
+      mode tcp
+      option tcp-check
+      tcp-check connect
+      {{- if .Values.auth }}
+      tcp-check send AUTH\ {{ .Values.redisPassword }}\r\n
+      tcp-check expect string +OK
+      {{- end }}
+      tcp-check send PING\r\n
+      tcp-check expect string +PONG
+      tcp-check send info\ replication\r\n
+      tcp-check expect string role:master
+      tcp-check send QUIT\r\n
+      tcp-check expect string +OK
+      {{- range $i := until $replicas }}
+      use-server R{{ $i }} if { srv_is_up(R{{ $i }}) } { nbsrv(check_if_redis_is_master_{{ $i }}) ge 2 }
+      server R{{ $i }} {{ $fullName }}-announce-{{ $i }}:6379 check inter 1s fall 1 rise 1
+      {{- end }}
+{{- end }}
+{{- end }}
+  haproxy_init.sh: | 
+    HAPROXY_CONF=/data/haproxy.cfg
+    cp /readonly/haproxy.cfg "$HAPROXY_CONF"
+    {{- $fullName := include "redis-ha.fullname" . }}
+    {{- $replicas := int .Values.replicas }}
+    {{- range $i := until $replicas }}
+    ANNOUNCE_IP{{ $i }}=$(getent hosts "{{ $fullName }}-announce-{{ $i }}" | awk '{ print $1 }')
+    sed -i "s/REPLACE_ANNOUNCE{{ $i }}/$ANNOUNCE_IP{{ $i }}/" "$HAPROXY_CONF"
+    {{- end }}
diff --git a/stable/redis-ha/templates/redis-ha-service.yaml b/stable/redis-ha/templates/redis-ha-service.yaml
@@ -28,4 +28,4 @@ spec:
 {{- end }}
   selector:
     release: {{ .Release.Name }}
-    app: {{ template "redis-ha.name" . }}
+    app: {{ template "redis-ha.name" . }}
diff --git a/stable/redis-ha/templates/redis-haproxy-deployment.yaml b/stable/redis-ha/templates/redis-haproxy-deployment.yaml
@@ -0,0 +1,94 @@
+{{- if .Values.haproxy.enabled }}
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: {{ template "redis-ha.fullname" . }}-haproxy
+  labels:
+{{ include "labels.standard" . | indent 4 }}
+spec:
+  strategy:
+    type: RollingUpdate
+  revisionHistoryLimit: 1
+  replicas: {{ .Values.haproxy.replicas }}
+  selector:
+    matchLabels:
+      app: {{ template "redis-ha.name" . }}-haproxy
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      name: {{ template "redis-ha.fullname" . }}-haproxy
+      labels:
+        app: {{ template "redis-ha.name" . }}-haproxy
+        release: {{ .Release.Name }}
+      annotations:
+        prometheus.io/path: /metrics
+        prometheus.io/port: "9101"
+        prometheus.io/scrape: "true"
+        checksum/config: {{ include (print $.Template.BasePath "/redis-ha-configmap.yaml") . | sha256sum }}
+      {{- if .Values.haproxy.annotations }}
+{{ toYaml .Values.haproxy.annotations . | indent 8 }}
+      {{- end }}
+    spec:
+      # Needed when using unmodified rbac-setup.yml
+      serviceAccountName: {{ template "redis-ha.serviceAccountName" . }}-haproxy
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+      initContainers:
+      - name: config-init
+        image: {{ .Values.haproxy.image.repository }}:{{ .Values.haproxy.image.tag }}
+        imagePullPolicy: {{ .Values.haproxy.image.pullPolicy }}
+        resources:
+{{ toYaml .Values.haproxy.init.resources | indent 10 }}
+        command:
+        - sh
+        args:
+        - /readonly/haproxy_init.sh
+        volumeMounts:
+        - name: config-volume
+          mountPath: /readonly
+          readOnly: true
+        - name: data
+          mountPath: /data
+      containers:
+      {{- if .Values.haproxy.exporter.enabled }}
+      - name: prometheus-exporter
+        image: {{ .Values.haproxy.exporter.image.repository }}:{{ .Values.haproxy.exporter.image.tag }}
+        imagePullPolicy: {{ .Values.haproxy.pullPolicy }}
+        ports:
+        - name: exporter-port
+          containerPort: {{ default "9101" .Values.haproxy.exporter.port }}
+        command: ["haproxy_exporter",
+                    "--haproxy.scrape-uri=unix:/run/haproxy/admin.sock"]
+        volumeMounts:
+          - name: shared-socket
+            mountPath: /run/haproxy
+      {{- end }}
+      - name: haproxy
+        image: {{ .Values.haproxy.image.repository }}:{{ .Values.haproxy.image.tag }}
+        imagePullPolicy: {{ .Values.haproxy.image.pullPolicy }}
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8888
+          initialDelaySeconds: 5
+          periodSeconds: 3
+        ports:
+        - name: redis
+          containerPort: {{ default "6379" .Values.redis.port }}
+          hostPort: 6379
+        resources:
+{{ toYaml .Values.haproxy.resources | indent 10 }}
+        volumeMounts:
+        - name: data
+          mountPath: /usr/local/etc/haproxy
+        - name: shared-socket
+          mountPath: /run/haproxy
+      volumes:
+      - name: config-volume
+        configMap:
+          name: {{ template "redis-ha.fullname" . }}-configmap
+      - name: shared-socket
+        emptyDir: {}
+      - name: data
+        emptyDir: {}
+{{- end }}
diff --git a/stable/redis-ha/templates/redis-haproxy-service.yaml b/stable/redis-ha/templates/redis-haproxy-service.yaml
@@ -0,0 +1,28 @@
+{{- if .Values.haproxy.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "redis-ha.fullname" . }}-haproxy
+  labels:
+{{ include "labels.standard" . | indent 4 }}
+  annotations:
+  {{- if .Values.haproxy.service.annotations }}
+{{ toYaml .Values.haproxy.service.annotations | indent 4 }}
+  {{- end }}
+spec:
+  type: {{ default "ClusterIP" .Values.haproxy.service.type }}
+  ports:
+  - name: haproxy
+    port: {{ .Values.redis.port }}
+    protocol: TCP
+    targetPort: redis
+{{- if .Values.exporter.enabled }}
+  - name: exporter-port
+    port: {{ .Values.haproxy.exporter.port }}
+    protocol: TCP
+    targetPort: exporter-port
+{{- end }}
+  selector:
+    release: {{ .Release.Name }}
+    app: {{ template "redis-ha.name" . }}-haproxy
+{{- end }}
diff --git a/stable/redis-ha/templates/redis-haproxy-serviceaccount.yaml b/stable/redis-ha/templates/redis-haproxy-serviceaccount.yaml
@@ -0,0 +1,11 @@
+{{- if and .Values.haproxy.serviceAccount.create .Values.haproxy.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "redis-ha.serviceAccountName" . }}-haproxy
+  labels:
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    app: {{ template "redis-ha.fullname" . }}
+{{- end }}
diff --git a/...emplates/tests/test-redis-ha-service.yaml → ...ha/templates/tests/test-redis-ha-pod.yaml b/...emplates/tests/test-redis-ha-service.yaml → ...ha/templates/tests/test-redis-ha-pod.yaml
diff --git a/stable/redis-ha/values.yaml b/stable/redis-ha/values.yaml
@@ -21,10 +21,40 @@ serviceAccount:
   ## If not set and create is true, a name is generated using the redis-ha.fullname template
   # name:
 
+## Enables a HA Proxy for better LoadBalancing / Sentinel Master support. Automatically proxies to Redis master.
+## Recommend for externally exposed Redis clusters.
+## ref: https://cbonte.github.io/haproxy-dconv/1.9/intro.html
+haproxy:
+  enabled: false
+  replicas: 1
+  image:
+    repository: haproxy
+    tag: 2.0.4
+    pullPolicy: IfNotPresent
+  annotations: {}
+  resources: {}
+  ## Service type for HAProxy
+  ##
+  service:
+    type: ClusterIP
+    annotations: {}
+  serviceAccount:
+    create: true
+  ## Prometheus metric exporter for HAProxy.
+  ##
+  exporter:
+    image:
+      repository: quay.io/prometheus/haproxy-exporter
+      tag: v0.9.0
+    enabled: false
+    port: 9101
+  init:
+    resources: {}
+
+
 ## Role Based Access
 ## Ref: https://kubernetes.io/docs/admin/authorization/rbac/
 ##
-
 rbac:
   create: true
 
@@ -108,6 +138,7 @@ securityContext:
 ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
 ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature
 ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+nodeSelector: {}
 
 ## Whether the Redis server pods should be forced to run on separate nodes.
 ## This is accomplished by setting their AntiAffinity with requiredDuringSchedulingIgnoredDuringExecution as opposed to preferred.