This commit fixes rafting and routing to the leader issues.

Fixes #107

This commit makes a service for each pod by using the unique statefull
set name label. These services ensure that there is a cluster ip
reserved for each pod. The rafting uses these cluster ips.

Orchestrator will proxy/route traffic to its leader. So, the main
service can be used as entry point and all the trafic will be routed to
the leader.

See: https://github.com/github/orchestrator/blob/master/docs/configuration-raft.md
See: presslabs/docker-orchestrator#8
Signed-off-by: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com>

docs/orchestrator.md

@@ -17,10 +17,11 @@ The MySQL operator uses [orchestrator](https://github.com/github/orchestrator),
 
 ## Access the orchestrator
 
-To connect to the orchestrator dashboard you have to point the orchestrator port 3000 to your local machine. Ensure it's a healthy pod if using raft:
+The service `<release-name>-mysql-operator` exposes port 80. Via this port you will be able to talk to the orchestrator leader.
+You can either port fowrwad this service to localhost, or use a service of type load balancer or enable the ingress.
 
 ```shell
-kubectl port-forward mysql-operator-orchestrator-0 3000
+kubectl port-forward service/<release-name>-mysql-operator 8080:80
 ```
 
 Then type `localhost:3000` in a browser.

hack/charts/mysql-operator/templates/_helpers.tpl

@@ -46,7 +46,7 @@ Create the name of the service account to use
 {{- $fullname := include "mysql-operator.fullname" . -}}
 {{- $replicas := int .Values.replicas -}}
 {{- range $i := until $replicas -}}
-{{ $fullname }}-{{ $i }}.{{ $fullname }}-orc{{- if lt $i (sub $replicas 1) }},{{ end }}
+{{ $fullname }}-{{ $i }}-svc{{- if lt $i (sub $replicas 1) }},{{ end }}
 {{- end -}}
 {{- end -}}
 
@@ -60,4 +60,17 @@ Create the name of the service account to use
 
 {{- define "mysql-operator.orc-service-name" -}}
 {{ include "mysql-operator.fullname" . }}-orc
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "mysql-operator.labels" -}}
+app.kubernetes.io/name: {{ include "mysql-operator.name" . }}
+helm.sh/chart: {{ include "mysql-operator.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}

hack/charts/mysql-operator/templates/orc-config.yaml

@@ -8,7 +8,9 @@
 {{- $_ := set $conf "SQLite3DataFile" "/var/lib/orchestrator/orc.db" }}
 {{- $_ := set $conf "RaftEnabled" true }}
 {{- $_ := set $conf "RaftDataDir" "/var/lib/orchestrator" }}
-{{- $_ := set $conf "RaftBind" (printf "{{ .Env.HOSTNAME }}.%s-orc" $fullname) }}
+{{- $_ := set $conf "RaftAdvertise" "{{ .Env.HOSTNAME }}-svc" }}
+{{- $_ := set $conf "RaftBind" "{{ .Env.HOSTNAME }}"}}
+{{- $_ := set $conf "HTTPAdvertise" "http://{{ .Env.HOSTNAME }}-svc:80" }}
 {{- if eq 1 $replicas -}}
 {{- $_ := set $conf "RaftNodes" (list) }}
 {{- else -}}

hack/charts/mysql-operator/templates/orc-ingress.yaml

@@ -0,0 +1,36 @@
+{{- if .Values.orchestrator.ingress.enabled -}}
+        {{- $fullName := include "mysql-operator.fullname" . -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+    name: {{ $fullName }}
+    labels:
+        {{ include "mysql-operator.labels" . | indent 4 }}
+        {{- with .Values.orchestrator.ingress.annotations }}
+    annotations:
+        {{- toYaml . | nindent 4 }}
+        {{- end }}
+spec:
+        {{- if .Values.orchestrator.ingress.tls }}
+    tls:
+            {{- range .Values.orchestrator.ingress.tls }}
+    - hosts:
+              {{- range .hosts }}
+      - {{ . | quote }}
+            {{- end }}
+      secretName: {{ .secretName }}
+        {{- end }}
+        {{- end }}
+    rules:
+            {{- range .Values.orchestrator.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+          paths:
+                  {{- range .paths }}
+          - path: {{ . }}
+            backend:
+                serviceName: {{ $fullName }}
+                servicePort: http
+        {{- end }}
+        {{- end }}
+        {{- end }}

hack/charts/mysql-operator/templates/orc-service.yaml

@@ -1,20 +1,54 @@
+{{- $fullName := include "mysql-operator.fullname" . }}
+{{- $replicas := int .Values.replicas -}}
+{{- $chart := include "mysql-operator.chart" . }}
+{{- range $i := until $replicas -}}
+{{/*
+this service is needed for rafting. https://github.com/presslabs/mysql-operator/issues/107
+and for routing/proxying to the leader.
+*/}}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "mysql-operator.orc-service-name" . }}
+  name: "{{$fullName}}-{{$i}}-svc"
+  labels:
+    app: "{{$fullName}}"
+    chart: {{$chart}}
+    release: "{{ $.Release.Name }}"
+    heritage: "{{ $.Release.Service }}"
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+spec:
+  type: ClusterIP
+  ports:
+  - name: web
+    port: 80
+    targetPort: 3000
+  - name: raft
+    port: 10008
+    targetPort: 10008
+  selector:
+    statefulset.kubernetes.io/pod-name: {{$fullName}}-{{$i}}
+---
+{{end}}
+{{/*{orchestrator will make sure that this service always talks to the leader}*/}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ template "mysql-operator.fullname" . }}"
   labels:
     app: {{ template "mysql-operator.name" . }}
     chart: {{ template "mysql-operator.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  clusterIP: None
-  # orchestrator nodes are ready only when the raft is ready and if the service doesn't publish
-  # unready pods, pods will never get ready (deadlock)
-  publishNotReadyAddresses: true
-  ports:
-    - name: raft
-      port: 10008
-      targetPort: 10008
+  type: {{ .Values.orchestrator.service.type }}
   selector:
     app: {{ template "mysql-operator.name" . }}
+  ports:
+  - name: http
+    port: {{ .Values.orchestrator.service.port }}
+    protocol: TCP
+    targetPort: 3000
+          {{- if .Values.orchestrator.service.nodePort }}
+    nodePort: {{ .Values.orchestrator.service.nodePort }}
+        {{- end }}

hack/charts/mysql-operator/values.yaml

@@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-replicas: 1
+replicas: 3
 image: quay.io/presslabs/mysql-operator:latest
 sidecarImage: quay.io/presslabs/mysql-operator-sidecar:latest
 imagePullPolicy: IfNotPresent
@@ -37,7 +37,7 @@ resources: {}
 ## end up on the same node. Setting this to "soft" will use
 ## preferredDuringSchedulingIgnoredDuringExecution. If set to anything else,
 ## no anti-affinity rules will be configured.
-antiAffinity: "hard"
+antiAffinity: "none"
 
 extraArgs: []
 
@@ -72,6 +72,25 @@ orchestrator:
     #  cpu: 100m
     #  memory: 128Mi
 
+  service:
+    type: ClusterIP
+    port: 80
+    # nodePort: 3000
+
+  ingress:
+    enabled: false
+    annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+    hosts:
+    - host: chart-example.local
+      paths: []
+
+    tls: []
+    #  - secretName: chart-example-tls
+    #    hosts:
+    #      - chart-example.local
+
   persistence:
     enabled: true
     ## If defined, storageClassName: <storageClass>