Support updating configuraion on the fly (#479)

* Support updating configuraion on the fly

Signed-off-by: Aylei <rayingecho@gmail.com>

charts/tidb-cluster/templates/_helpers.tpl

@@ -25,4 +25,63 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 
 {{- define "cluster.name" -}}
 {{- default .Release.Name .Values.clusterName }}
-{{- end -}}
+{{- end -}}
+
+{{/*
+Encapsulate PD configmap data for consistent digest calculation
+*/}}
+{{- define "pd-configmap.data" -}}
+startup-script: |-
+{{ tuple "scripts/_start_pd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+config-file: |-
+    {{- if .Values.pd.config }}
+{{ .Values.pd.config | indent 2 }}
+    {{- else }}
+{{ tuple "config/_pd-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "pd-configmap.data-digest" -}}
+{{ include "pd-configmap.data" . | sha256sum | trunc 8 }}
+{{- end -}}
+
+{{/*
+Encapsulate TiKV configmap data for consistent digest calculation
+*/}}
+{{- define "tikv-configmap.data" -}}
+startup-script: |-
+{{ tuple "scripts/_start_tikv.sh.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+config-file: |-
+    {{- if .Values.tikv.config }}
+{{ .Values.tikv.config | indent 2 }}
+    {{- else }}
+{{ tuple "config/_tikv-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "tikv-configmap.data-digest" -}}
+{{ include "tikv-configmap.data" . | sha256sum | trunc 8 }}
+{{- end -}}
+
+{{/*
+Encapsulate TiDB configmap data for consistent digest calculation
+*/}}
+{{- define "tidb-configmap.data" -}}
+startup-script: |-
+{{ tuple "scripts/_start_tidb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+  {{- if .Values.tidb.initSql }}
+init-sql: |-
+{{ .Values.tidb.initSql | indent 2 }}
+  {{- end }}
+config-file: |-
+    {{- if .Values.tidb.config }}
+{{ .Values.tidb.config | indent 2 }}
+    {{- else }}
+{{ tuple "config/_tidb-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "tidb-configmap.data-digest" -}}
+{{ include "tidb-configmap.data" . | sha256sum | trunc 8 }}
+{{- end -}}
+

charts/tidb-cluster/templates/pd-configmap.yaml

@@ -1,20 +1,16 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
+{{- if .Values.enableConfigMapRollout }}
+  name: {{ template "cluster.name" . }}-pd-{{ template "pd-configmap.data-digest" . }}
+{{- else }}
   name: {{ template "cluster.name" . }}-pd
+{{- end }}
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: pd
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 data:
-  startup-script: |-
-{{ tuple "scripts/_start_pd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-
-  config-file: |-
-    {{- if .Values.pd.config }}
-{{ .Values.pd.config | indent 4 }}
-    {{- else }}
-{{ tuple "config/_pd-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-    {{- end -}}
+{{ include "pd-configmap.data" . | indent 2 }}

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -2,6 +2,12 @@ apiVersion: pingcap.com/v1alpha1
 kind: TidbCluster
 metadata:
   name: {{ template "cluster.name" . }}
+{{- if .Values.enableConfigMapRollout }}
+  annotations:
+    pingcap.com/pd.{{ template "cluster.name" . }}-pd.sha: {{ template "pd-configmap.data-digest" . }}
+    pingcap.com/tikv.{{ template "cluster.name" . }}-tikv.sha: {{ template "tikv-configmap.data-digest" . }}
+    pingcap.com/tidb.{{ template "cluster.name" . }}-tidb.sha: {{ template "tidb-configmap.data-digest" . }}
+{{- end }}
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}

charts/tidb-cluster/templates/tidb-configmap.yaml

@@ -1,23 +1,16 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
+{{- if .Values.enableConfigMapRollout }}
+  name: {{ template "cluster.name" . }}-tidb-{{ template "tidb-configmap.data-digest" . }}
+{{- else }}
   name: {{ template "cluster.name" . }}-tidb
+{{- end }}
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: tidb
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 data:
-  startup-script: |-
-{{ tuple "scripts/_start_tidb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  {{- if .Values.tidb.initSql }}
-  init-sql: |-
-{{ .Values.tidb.initSql | indent 4 }}
-  {{- end }}
-  config-file: |-
-    {{- if .Values.tidb.config }}
-{{ .Values.tidb.config | indent 4 }}
-    {{- else }}
-{{ tuple "config/_tidb-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-    {{- end -}}
+{{ include "tidb-configmap.data" . | indent 2 }}

charts/tidb-cluster/templates/tidb-initializer-job.yaml

@@ -3,6 +3,8 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: {{ template "cluster.name" . }}-tidb-initializer
+  annotations:
+    "helm.sh/hook": post-install
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
@@ -44,7 +46,11 @@ spec:
       {{- if .Values.tidb.initSql }}
       - name: init-sql
         configMap:
+          {{- if .Values.enableConfigMapRollout }}
+          name: {{ template "cluster.name" . }}-tidb-{{ template "tidb-configmap.data-digest" . }}
+          {{- else }}
           name: {{ template "cluster.name" . }}-tidb
+          {{- end }}
           items:
           - key: init-sql
             path: init.sql

charts/tidb-cluster/templates/tikv-configmap.yaml

@@ -1,20 +1,16 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
+{{- if .Values.enableConfigMapRollout }}
+  name: {{ template "cluster.name" . }}-tikv-{{ template "tikv-configmap.data-digest" . }}
+{{- else }}
   name: {{ template "cluster.name" . }}-tikv
+{{- end }}
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: tikv
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 data:
-  startup-script: |-
-{{ tuple "scripts/_start_tikv.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-
-  config-file: |-
-    {{- if .Values.tikv.config }}
-{{ .Values.tikv.config | indent 4 }}
-    {{- else }}
-{{ tuple "config/_tikv-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-    {{- end -}}
+{{ include "tikv-configmap.data" . | indent 2 }}

charts/tidb-cluster/values.yaml

@@ -40,6 +40,13 @@ discovery:
       cpu: 80m
       memory: 50Mi
 
+# Whether enable ConfigMap Rollout management.
+# When enabling, change of ConfigMap will trigger a graceful rolling-update of the component.
+# This feature is only available in tidb-operator v1.0 or higher.
+# Note: Switch this variable against an existing cluster will cause an rolling-update of each component even
+# if the ConfigMap was not changed.
+enableConfigMapRollout: false
+
 pd:
   replicas: 3
   image: pingcap/pd:v2.1.8

docs/operation-guide.md

@@ -104,6 +104,12 @@ $ helm upgrade ${releaseName} charts/tidb-cluster
 
 For minor version upgrade, updating the `image` should be enough. When TiDB major version is out, the better way to update is to fetch the new charts from tidb-operator and then merge the old values.yaml with new values.yaml. And then upgrade as above.
 
+## Change TiDB cluster Configuration
+
+Since `v1.0.0`, TiDB operator can perform rolling-update on configuration updates. This feature is disabled by default in favor of backward compatibility, you can enable it by setting `enableConfigMapRollout` to `true` in your helm values file.
+
+> WARN: changing this variable against a running cluster will trigger an rolling-update of PD/TiKV/TiDB pods even if there's no configuration change.
+
 ## Destroy TiDB cluster
 
 To destroy TiDB cluster, run the following command:

go.mod

@@ -61,12 +61,14 @@ require (
 	github.com/onsi/gomega v1.4.1
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pborman/uuid v1.2.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 // indirect
 	github.com/pingcap/errors v0.11.0
 	github.com/pingcap/kvproto v0.0.0-20180606093822-b7ba8ea1c0b4
 	github.com/pingcap/pd v2.1.0-beta+incompatible
+	github.com/pingcap/tidb v2.1.0-beta+incompatible
 	github.com/pkg/errors v0.8.0 // indirect
 	github.com/prometheus/client_golang v0.8.0
 	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
@@ -80,6 +82,8 @@ require (
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
+	github.com/uber/jaeger-client-go v2.16.0+incompatible // indirect
+	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect
 	github.com/ugorji/go v1.1.1 // indirect
 	github.com/unrolled/render v0.0.0-20180807193321-4206df6ff701 // indirect
 	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect

go.sum

@@ -148,6 +148,8 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
@@ -160,6 +162,8 @@ github.com/pingcap/kvproto v0.0.0-20180606093822-b7ba8ea1c0b4 h1:fYmrrTx2rWqgvlv
 github.com/pingcap/kvproto v0.0.0-20180606093822-b7ba8ea1c0b4/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk=
 github.com/pingcap/pd v2.1.0-beta+incompatible h1:DZrskt6POM+zhTZvUjUJKXV9OVWpV7cdcq1tXQGOiCY=
 github.com/pingcap/pd v2.1.0-beta+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
+github.com/pingcap/tidb v2.1.0-beta+incompatible h1:SQUmscnvvjHLjaIycQqtHujBahUnlKwTz6dQQhqgGSc=
+github.com/pingcap/tidb v2.1.0-beta+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -192,6 +196,10 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
+github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw=
+github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w=
 github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/unrolled/render v0.0.0-20180807193321-4206df6ff701 h1:BJ/T25enw0WcbWqV132hGXRQdqCqe9XBzqh4AWVH7Bc=

pkg/controller/controller_utils.go

@@ -155,6 +155,24 @@ func AnnProm(port int32) map[string]string {
 	}
 }
 
+// MemberConfigMapName returns the default ConfigMap name of the specified member type
+func MemberConfigMapName(tc *v1alpha1.TidbCluster, member v1alpha1.MemberType) string {
+	nameKey := fmt.Sprintf("%s-%s", tc.Name, member)
+	return nameKey + getConfigMapSuffix(tc, member.String(), nameKey)
+}
+
+// getConfigMapSuffix return the ConfigMap name suffix
+func getConfigMapSuffix(tc *v1alpha1.TidbCluster, component string, name string) string {
+	if tc.Annotations == nil {
+		return ""
+	}
+	sha := tc.Annotations[fmt.Sprintf("pingcap.com/%s.%s.sha", component, name)]
+	if len(sha) == 0 {
+		return ""
+	}
+	return "-" + sha
+}
+
 // setIfNotEmpty set the value into map when value in not empty
 func setIfNotEmpty(container map[string]string, key, value string) {
 	if value != "" {

pkg/controller/controller_utils_test.go

@@ -185,6 +185,82 @@ func TestAnnProm(t *testing.T) {
 	g.Expect(ann["prometheus.io/port"]).To(Equal("9090"))
 }
 
+func TestMemberConfigMapName(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	type testcase struct {
+		name        string
+		annotations map[string]string
+		tcName      string
+		member      v1alpha1.MemberType
+		expectFn    func(*GomegaWithT, string)
+	}
+	testFn := func(test *testcase, t *testing.T) {
+		t.Log(test.name)
+		tc := &v1alpha1.TidbCluster{}
+		tc.Name = test.tcName
+		tc.Annotations = test.annotations
+		test.expectFn(g, MemberConfigMapName(tc, test.member))
+	}
+	tests := []testcase{
+		{
+			name:        "backward compatible when no annotations set",
+			annotations: map[string]string{},
+			tcName:      "cluster-name",
+			member:      v1alpha1.TiKVMemberType,
+			expectFn: func(g *GomegaWithT, s string) {
+				g.Expect(s).To(Equal("cluster-name-tikv"))
+			},
+		},
+		{
+			name: "configmap digest presented",
+			annotations: map[string]string{
+				"pingcap.com/tikv.cluster-name-tikv.sha": "uuuuuuuu",
+			},
+			tcName: "cluster-name",
+			member: v1alpha1.TiKVMemberType,
+			expectFn: func(g *GomegaWithT, s string) {
+				g.Expect(s).To(Equal("cluster-name-tikv-uuuuuuuu"))
+			},
+		},
+		{
+			name:        "nil annotations",
+			annotations: nil,
+			tcName:      "cluster-name",
+			member:      v1alpha1.TiKVMemberType,
+			expectFn: func(g *GomegaWithT, s string) {
+				g.Expect(s).To(Equal("cluster-name-tikv"))
+			},
+		},
+		{
+			name: "annotation presented with empty value empty",
+			annotations: map[string]string{
+				"pingcap.com/tikv.cluster-name-tikv.sha": "",
+			},
+			tcName: "cluster-name",
+			member: v1alpha1.TiKVMemberType,
+			expectFn: func(g *GomegaWithT, s string) {
+				g.Expect(s).To(Equal("cluster-name-tikv"))
+			},
+		},
+		{
+			name: "no matched annotation key",
+			annotations: map[string]string{
+				"pingcap.com/pd.cluster-name-tikv.sha": "",
+			},
+			tcName: "cluster-name",
+			member: v1alpha1.TiKVMemberType,
+			expectFn: func(g *GomegaWithT, s string) {
+				g.Expect(s).To(Equal("cluster-name-tikv"))
+			},
+		},
+	}
+
+	for i := range tests {
+		testFn(&tests[i], t)
+	}
+}
+
 func TestSetIfNotEmpty(t *testing.T) {
 	g := NewGomegaWithT(t)