#6 - ensures better resiliency and moves joining to initialisation.

cleanup

@@ -1,9 +1,11 @@
 #!/bin/sh
 
-rm ca-key.pem ca.csr ca.pem consul-key.pem consul.csr consul.pem
-kubectl delete statefulset consul
-kubectl delete pvc data-consul-0 data-consul-1 data-consul-2
-kubectl delete svc consul
-kubectl delete jobs consul-join
-kubectl delete secrets consul
-kubectl delete configmaps consul
+rm -fv ca-key.pem ca.csr ca.pem consul-key.pem consul.csr consul.pem
+kubectl delete statefulset consul  > /dev/null 2>&1;
+kubectl get pvc --no-headers | awk '/data-consul/ {print $1}' | xargs --no-run-if-empty kubectl delete pvc
+kubectl get pv --no-headers | awk '/data-consul/ {print $1}' | xargs --no-run-if-empty kubectl delete pv
+kubectl delete svc consul > /dev/null 2>&1;
+kubectl delete jobs consul-join  > /dev/null 2>&1;
+kubectl delete secrets consul  > /dev/null 2>&1;
+kubectl delete configmaps consul > /dev/null 2>&1;
+

configs/server.json

@@ -7,5 +7,6 @@
   "verify_server_hostname": true,
   "ports": {
     "https": 8443
-  }
+  },
+  "rejoin_after_leave": true
 }

jobs/consul-join.yaml

@@ -22,4 +22,6 @@ spec:
             - "-rpc-addr=consul-0.consul.$(NAMESPACE).svc.cluster.local:8400"
             - consul-1.consul.$(NAMESPACE).svc.cluster.local
             - consul-2.consul.$(NAMESPACE).svc.cluster.local
+            - consul-3.consul.$(NAMESPACE).svc.cluster.local
+            - consul-4.consul.$(NAMESPACE).svc.cluster.local
       restartPolicy: Never

setup

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+cfssl gencert -initca ca/ca-csr.json | cfssljson -bare ca
+cfssl gencert \
+  -ca=ca.pem \
+  -ca-key=ca-key.pem \
+  -config=ca/ca-config.json \
+  -profile=default \
+  ca/consul-csr.json | cfssljson -bare consul;
+
+GOSSIP_ENCRYPTION_KEY=$(consul keygen);
+echo GOSSIP_ENCRYPTION_KEY - $GOSSIP_ENCRYPTION_KEY
+kubectl create secret generic consul \
+  --from-literal="gossip-encryption-key=${GOSSIP_ENCRYPTION_KEY}" \
+  --from-file=ca.pem \
+  --from-file=consul.pem \
+  --from-file=consul-key.pem
+
+kubectl create configmap consul --from-file=configs/server.json;
+
+kubectl create -f services/consul.yaml;
+
+kubectl create -f statefulsets/consul.yaml;
+
+kubectl get pods -owide;
+
+
+while [ 1 ]; do 
+    stateful_up=$(kubectl get --no-headers statefulsets consul -owide 2>/dev/null | awk '{ if ($2 != $3) { print 0 } else { print 1; } }' 2>/dev/null);
+    if [ "${stateful_up}" == "1" ]; then
+        printf 'consul stateful service up\n'
+        break;
+    else
+        printf '.';
+        sleep 1;
+    fi
+done
+
+echo;
+
+function get_status {
+    local conul_num=${1:-0};
+    kubectl get pods -owide;
+    kubectl exec --tty -i consul-${conul_num} -- consul members -detailed;
+    kubectl exec --tty -i consul-${conul_num} -- consul operator raft -list-peers;
+}
+
+get_status;
+
+
+#kubectl create -f jobs/consul-join.yaml

statefulsets/consul.yaml

@@ -4,7 +4,7 @@ metadata:
   name: consul
 spec:
   serviceName: consul
-  replicas: 3
+  replicas: 5
   template:
     metadata:
       labels:
@@ -17,28 +17,70 @@ spec:
         - name: consul
           image: "consul:0.7.2"
           env:
+            - name: INITIAL_CLUSTER_SIZE
+              value: "3"
             - name: POD_IP
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: STATEFULSET_NAME
+              value: consul
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             - name: GOSSIP_ENCRYPTION_KEY
               valueFrom:
                 secretKeyRef:
                   name: consul
                   key: gossip-encryption-key
-          args:
-            - "agent"
-            - "-advertise=$(POD_IP)"
-            - "-bind=0.0.0.0"
-            - "-bootstrap-expect=3"
-            - "-client=0.0.0.0"
-            - "-config-file=/etc/consul/server.json"
-            - "-datacenter=dc1"
-            - "-data-dir=/var/lib/consul"
-            - "-domain=cluster.local"
-            - "-encrypt=$(GOSSIP_ENCRYPTION_KEY)"
-            - "-server"
-            - "-ui"
+          lifecycle:
+            preStop:
+              exec:
+                command: ["consul","leave"]
+          command:
+            - "/bin/sh"
+            - "-exc"
+            - |
+              if [ -e /etc/consul/secrets/gossip-key ]; then
+                echo "{\"encrypt\": \"$(base64 /etc/consul/secrets/gossip-key)\"}" > /etc/consul/encrypt.json
+                GOSSIP_KEY="-config-file /etc/consul/encrypt.json"
+              fi
+              for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
+                while true; do
+                    echo "Waiting for ${STATEFULSET_NAME}-${i}.${STATEFULSET_NAME} to come up"
+                    ping -W 1 -c 1 ${STATEFULSET_NAME}-${i}.${STATEFULSET_NAME}.${NAMESPACE}.svc.cluster.local > /dev/null && break
+                    sleep 1s
+                done
+              done
+              PEERS=""
+              PEERS_JSON="["
+              for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
+                  PEERS="${PEERS}${PEERS:+ } -retry-join ${STATEFULSET_NAME}-${i}.${STATEFULSET_NAME}.${NAMESPACE}.svc.cluster.local"
+                  PEERS_JSON="${PEERS_JSON}\"${STATEFULSET_NAME}-${i}.${STATEFULSET_NAME}.${NAMESPACE}.svc.cluster.local\","
+              done
+              PEERS_JSON=$(echo "${PEERS_JSON}" | sed 's/,$/]/');
+              if [ -e "/var/lib/consul/raft/" ]; then
+                  echo "${PEERS_JSON}" >> /var/lib/consul/raft/peers.json;
+              fi
+              exec /bin/consul agent \
+                -data-dir=/var/lib/consul \
+                -server \
+                -ui \
+                -bootstrap-expect=${INITIAL_CLUSTER_SIZE} \
+                -advertise=${POD_IP} \
+                -config-file=/etc/consul/server.json \
+                -datacenter=dc1 \
+                -domain=cluster.local \
+                -node=${POD_NAME} \
+                ${PEERS} \
+                ${GOSSIP_KEY} \
+                -bind=0.0.0.0 \
+                -client=0.0.0.0
           volumeMounts:
             - name: data
               mountPath: /var/lib/consul

test

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+function get_status {
+    local consul_num=${1:-0};
+    kubectl get pods -owide;
+    kubectl exec --tty -i consul-${consul_num} -- consul members -detailed;
+    kubectl exec --tty -i consul-${consul_num} -- consul operator raft -list-peers -stale=true;
+    kubectl exec --tty -i consul-${consul_num} -- consul info;
+
+    kubectl exec --tty -i consul-${consul_num} -- consul kv put test 1
+}
+
+get_status;
+exit;
+kubectl delete pods consul-1 consul-2 consul-3;
+sleep 1;
+get_status;
+
+sleep 20;
+
+get_status;
+
+kubectl patch statefulset/consul -p '{"spec":{"replicas": 3}}'
+sleep 5;
+
+get_status;
+
+addr=$(kubectl exec --tty -i consul-0 -- consul operator raft -list-peers | awk '/leader/ {print $2}');