Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

make tidb-initializer support TLS #1931

Merged
merged 10 commits into from
Mar 18, 2020
Merged

make tidb-initializer support TLS #1931

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
830f9c3
let tidb-initializer support TLS
weekface Mar 16, 2020
05e82bb
address comment
weekface Mar 16, 2020
d07f9cd
use should create client certificate when tlsClient set true
weekface Mar 16, 2020
71afb43
Merge branch 'master' into tidb-initializer-tls
DanielZhangQD Mar 16, 2020
d13ef43
fix ci
weekface Mar 17, 2020
8bc9745
Merge branch 'master' into tidb-initializer-tls
weekface Mar 17, 2020
8abb419
Merge branch 'master' into tidb-initializer-tls
weekface Mar 18, 2020
68df42e
Merge branch 'master' into tidb-initializer-tls
DanielZhangQD Mar 18, 2020
27e18c4
Merge branch 'master' into tidb-initializer-tls
sre-bot Mar 18, 2020
c42c0d4
Merge branch 'master' into tidb-initializer-tls
sre-bot Mar 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion charts/tidb-cluster/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,10 @@ tidb:
# 2. Create a K8s Secret object which contains the TiDB server-side certificate created above.
# The name of this Secret must be: <clusterName>-tidb-server-secret.
# kubectl create secret generic <clusterName>-tidb-server-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
# 3. Then create the TiDB cluster with `tlsClient.enabled` set to `true`.
# 3. Create a K8s Secret object which contains the TiDB client-side certificate created above which will be used by TiDB Operator.
# The name of this Secret must be: <clusterName>-tidb-client-secret.
# kubectl create secret generic <clusterName>-tidb-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
# 4. Then create the TiDB cluster with `tlsClient.enabled` set to `true`.
enabled: false

# mysqlClient is used to set password for TiDB
Expand Down
5 changes: 4 additions & 1 deletion docs/api-references/docs.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7665,7 +7665,10 @@ <h3 id="pingcap.com/v1alpha1.TiDBTLSClient">TiDBTLSClient
2. Create a K8s Secret object which contains the TiDB server-side certificate created above.
The name of this Secret must be: <clusterName>-tidb-server-secret.
kubectl create secret generic <clusterName>-tidb-server-secret &ndash;namespace=<namespace> &ndash;from-file=tls.crt=<path/to/tls.crt> &ndash;from-file=tls.key=<path/to/tls.key> &ndash;from-file=ca.crt=<path/to/ca.crt>
3. Set Enabled to <code>true</code>.</p>
3. Create a K8s Secret object which contains the TiDB client-side certificate created above which will be used by TiDB Operator.
The name of this Secret must be: <clusterName>-tidb-client-secret.
kubectl create secret generic <clusterName>-tidb-client-secret &ndash;namespace=<namespace> &ndash;from-file=tls.crt=<path/to/tls.crt> &ndash;from-file=tls.key=<path/to/tls.key> &ndash;from-file=ca.crt=<path/to/ca.crt>
4. Set Enabled to <code>true</code>.</p>
</td>
</tr>
</tbody>
Expand Down
5 changes: 4 additions & 1 deletion pkg/apis/pingcap/v1alpha1/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,7 +616,10 @@ type TiDBTLSClient struct {
// 2. Create a K8s Secret object which contains the TiDB server-side certificate created above.
// The name of this Secret must be: <clusterName>-tidb-server-secret.
// kubectl create secret generic <clusterName>-tidb-server-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
// 3. Set Enabled to `true`.
// 3. Create a K8s Secret object which contains the TiDB client-side certificate created above which will be used by TiDB Operator.
// The name of this Secret must be: <clusterName>-tidb-client-secret.
// kubectl create secret generic <clusterName>-tidb-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
// 4. Set Enabled to `true`.
// +optional
Enabled bool `json:"enabled,omitempty"`
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/controller/tidbinitializer/tidb_initializer_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ func NewController(
recorder := eventBroadcaster.NewRecorder(v1alpha1.Scheme, corev1.EventSource{Component: "tidbinitializer"})

tidbInitializerInformer := informerFactory.Pingcap().V1alpha1().TidbInitializers()
tidbClusterInformer := informerFactory.Pingcap().V1alpha1().TidbClusters()
jobInformer := kubeInformerFactory.Batch().V1().Jobs()
typedControl := controller.NewTypedControl(controller.NewRealGenericControl(genericCli, recorder))

Expand All @@ -75,6 +76,7 @@ func NewController(
jobInformer.Lister(),
genericCli,
tidbInitializerInformer.Lister(),
tidbClusterInformer.Lister(),
typedControl,
),
),
Expand Down
8 changes: 8 additions & 0 deletions pkg/manager/member/template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,11 @@ var tidbInitStartScriptTpl = template.Must(template.New("tidb-init-start-script"
host = '{{ .ClusterName }}-tidb'
permit_host = '{{ .PermitHost }}'
port = 4000
{{- if .TLS }}
conn = MySQLdb.connect(host=host, port=port, user='root', connect_timeout=5, ssl={'ca': '{{ .CAPath }}', 'cert': '{{ .CertPath }}', 'key': '{{ .KeyPath }}'})
Copy link
Contributor

@cofyc cofyc Mar 16, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as MySQL client, it can connect a TiDB server with tls enabled without client certificate (--ssl/--ssl-mode in mysql CLI). the communication is SSL encrypted too. tidb server does not verify the client certificate and has no plan to verify it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tidb server does not verify the client certificate and has no plan to verify it.

Not exactly, TiDB Server already supported Certificate-Based Authentication for Login.

With this method, the TiDB Server verifies client certificates when users log in.

Copy link
Contributor

@cofyc cofyc Mar 16, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if users enabled tidb server-side TLS, they must provide client tls secret for the root user to initialize the tidb cluster? IMO, this is complex and the initialization will fail if the user didn't configure this secret. Maybe we can add a dedicate client secret and a switch for the tidb initializer?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can configure a tls client secret like TidbInitializerSpec.PasswordSecret for the tidb initializer user when people want to initialize the cluster with the client secret provided. and it's optional because when the server is TLS enabled, people can still connect the server without SSL certificate.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, when TLS was enabled, the user should prepare two certificates: one server cert and one client cert, we will have document to mention that.

The client certificate Secret Name is <clusterName>-tidb-client-secret.

{{- else }}
conn = MySQLdb.connect(host=host, port=port, user='root', connect_timeout=5)
{{- end }}
{{- if .PasswordSet }}
password_dir = '/etc/tidb/password'
for file in os.listdir(password_dir):
Expand Down Expand Up @@ -313,6 +317,10 @@ type TiDBInitStartScriptModel struct {
PermitHost string
PasswordSet bool
InitSQL bool
TLS bool
CAPath string
CertPath string
KeyPath string
}

func RenderTiDBInitStartScript(model *TiDBInitStartScriptModel) (string, error) {
Expand Down
47 changes: 43 additions & 4 deletions pkg/manager/member/tidb_init_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/pkg/util"
)

const (
Expand Down Expand Up @@ -59,6 +60,7 @@ type tidbInitManager struct {
jobLister batchlisters.JobLister
genericCli client.Client
tiLister listers.TidbInitializerLister
tcLister listers.TidbClusterLister
typedControl controller.TypedControlInterface
}

Expand All @@ -67,12 +69,14 @@ func NewTiDBInitManager(
jobLister batchlisters.JobLister,
genericCli client.Client,
tiLister listers.TidbInitializerLister,
tcLister listers.TidbClusterLister,
typedControl controller.TypedControlInterface,
) InitManager {
return &tidbInitManager{
jobLister,
genericCli,
tiLister,
tcLister,
typedControl,
}
}
Expand Down Expand Up @@ -134,6 +138,7 @@ func (tm *tidbInitManager) syncTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) e
name := controller.TiDBInitializerMemberName(ti.Spec.Clusters.Name)
ns := ti.Namespace
cm := &corev1.ConfigMap{}
tcName := ti.Spec.Clusters.Name

exist, err := tm.typedControl.Exist(client.ObjectKey{
Namespace: ns,
Expand All @@ -146,7 +151,12 @@ func (tm *tidbInitManager) syncTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) e
return nil
}

newCm, err := getTiDBInitConfigMap(ti)
tc, err := tm.tcLister.TidbClusters(ns).Get(tcName)
if err != nil {
return err
}

newCm, err := getTiDBInitConfigMap(ti, tc.Spec.TiDB.IsTLSClientEnabled())
if err != nil {
return err
}
Expand Down Expand Up @@ -187,6 +197,13 @@ func (tm *tidbInitManager) syncTiDBInitJob(ti *v1alpha1.TidbInitializer) error {

func (tm *tidbInitManager) makeTiDBInitJob(ti *v1alpha1.TidbInitializer) (*batchv1.Job, error) {
jobName := controller.TiDBInitializerMemberName(ti.Spec.Clusters.Name)
ns := ti.Namespace
tcName := ti.Spec.Clusters.Name

tc, err := tm.tcLister.TidbClusters(ns).Get(tcName)
if err != nil {
return nil, err
}

var envs []corev1.EnvVar
if ti.Spec.Timezone != "" {
Expand All @@ -207,6 +224,21 @@ func (tm *tidbInitManager) makeTiDBInitJob(ti *v1alpha1.TidbInitializer) (*batch

var vms []corev1.VolumeMount
var vs []corev1.Volume
if tc.Spec.TiDB.IsTLSClientEnabled() {
vms = append(vms, corev1.VolumeMount{
Name: "tidb-client-tls",
ReadOnly: true,
MountPath: util.TiDBClientTLSPath,
})
vs = append(vs, corev1.Volume{
Name: "tidb-client-tls",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: util.TiDBClientTLSSecretName(tcName),
},
},
})
}
vms = append(vms, corev1.VolumeMount{
Name: startKey,
ReadOnly: true,
Expand Down Expand Up @@ -335,7 +367,7 @@ func (tm *tidbInitManager) makeTiDBInitJob(ti *v1alpha1.TidbInitializer) (*batch
return job, nil
}

func getTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) (*corev1.ConfigMap, error) {
func getTiDBInitConfigMap(ti *v1alpha1.TidbInitializer, tlsClientEnabled bool) (*corev1.ConfigMap, error) {
var initSQL, passwdSet bool

permitHost := ti.GetPermitHost()
Expand All @@ -354,12 +386,19 @@ func getTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) (*corev1.ConfigMap, erro
return nil, err
}

startScript, err := RenderTiDBInitStartScript(&TiDBInitStartScriptModel{
initModel := &TiDBInitStartScriptModel{
ClusterName: ti.Spec.Clusters.Name,
PermitHost: permitHost,
InitSQL: initSQL,
PasswordSet: passwdSet,
})
}
if tlsClientEnabled {
initModel.TLS = true
initModel.CAPath = path.Join(util.TiDBClientTLSPath, corev1.ServiceAccountRootCAKey)
initModel.CertPath = path.Join(util.TiDBClientTLSPath, corev1.TLSCertKey)
initModel.KeyPath = path.Join(util.TiDBClientTLSPath, corev1.TLSPrivateKeyKey)
}
startScript, err := RenderTiDBInitStartScript(initModel)
if err != nil {
return nil, err
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/util/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

var (
ClusterClientTLSPath = "/var/lib/cluster-client-tls"
TiDBClientTLSPath = "/var/lib/tidb-client-tls"
)

func GetOrdinalFromPodName(podName string) (int32, error) {
Expand Down Expand Up @@ -175,3 +176,7 @@ func ClusterClientTLSSecretName(tcName string) string {
func ClusterTLSSecretName(tcName, component string) string {
return fmt.Sprintf("%s-%s-cluster-secret", tcName, component)
}

func TiDBClientTLSSecretName(tcName string) string {
return fmt.Sprintf("%s-tidb-client-secret", tcName)
}