From 606774b8acf139be31acbdc63a4b58bee6d5ac7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=B9=E4=BA=AE?= <30903849+shuijing198799@users.noreply.github.com> Date: Fri, 13 Mar 2020 20:48:54 +0800 Subject: [PATCH] backup: support kms decryption secret (#1908) --- images/tidb-backup-manager/Dockerfile | 5 +++++ images/tidb-backup-manager/entrypoint.sh | 17 ++++++++++++----- manifests/backup/backup-aws-s3-br.yaml | 1 + manifests/backup/backup-s3-br.yaml | 1 + manifests/backup/backup-schedule-aws-s3-br.yaml | 1 + manifests/backup/backup-schedule-s3-br.yaml | 1 + manifests/backup/restore-aws-s3-br.yaml | 1 + manifests/backup/restore-s3-br.yaml | 1 + manifests/crd.yaml | 9 +++++++++ pkg/apis/pingcap/v1alpha1/openapi_generated.go | 14 ++++++++++++++ pkg/apis/pingcap/v1alpha1/types.go | 4 ++++ pkg/backup/backup/backup_cleaner.go | 1 - pkg/backup/backup/backup_manager.go | 4 ++-- pkg/backup/constants/constants.go | 3 +++ pkg/backup/restore/restore_manager.go | 4 ++-- pkg/backup/util/util.go | 12 ++++++++++-- 16 files changed, 67 insertions(+), 12 deletions(-) diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index c7ae4a4caf..17fdc3e216 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -1,5 +1,6 @@ FROM pingcap/tidb-enterprise-tools:latest ARG VERSION=v1.51.0 +ARG SHUSH_VERSION=v1.4.0 RUN apk update && apk add ca-certificates RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \ @@ -14,6 +15,10 @@ RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ && chmod 755 /usr/local/bin/br \ && rm -rf br-latest-linux-amd64.tar.gz +RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \ + && mv shush_linux_amd64 /usr/local/bin/shush \ + && chmod 755 /usr/local/bin/shush + COPY bin/tidb-backup-manager /tidb-backup-manager COPY entrypoint.sh /entrypoint.sh diff --git a/images/tidb-backup-manager/entrypoint.sh b/images/tidb-backup-manager/entrypoint.sh index 85c889147d..427139451e 100755 --- a/images/tidb-backup-manager/entrypoint.sh +++ b/images/tidb-backup-manager/entrypoint.sh @@ -51,33 +51,40 @@ else fi BACKUP_BIN=/tidb-backup-manager +if [[ -n "${AWS_DEFAULT_REGION}"]]; then + EXEC_COMMAND="exec" +else + EXEC_COMMAND="/usr/local/bin/shush exec --" +fi + +cat /tmp/rclone.conf # exec command case "$1" in backup) shift 1 echo "$BACKUP_BIN backup $@" - exec $BACKUP_BIN backup "$@" + $EXEC_COMMAND $BACKUP_BIN backup "$@" ;; export) shift 1 echo "$BACKUP_BIN export $@" - exec $BACKUP_BIN export "$@" + $EXEC_COMMAND $BACKUP_BIN export "$@" ;; restore) shift 1 echo "$BACKUP_BIN restore $@" - exec $BACKUP_BIN restore "$@" + $EXEC_COMMAND $BACKUP_BIN restore "$@" ;; import) shift 1 echo "$BACKUP_BIN import $@" - exec $BACKUP_BIN import "$@" + $EXEC_COMMAND $BACKUP_BIN import "$@" ;; clean) shift 1 echo "$BACKUP_BIN clean $@" - exec $BACKUP_BIN clean "$@" + $EXEC_COMMAND $BACKUP_BIN clean "$@" ;; *) echo "Usage: $0 {backup|restore|clean}" diff --git a/manifests/backup/backup-aws-s3-br.yaml b/manifests/backup/backup-aws-s3-br.yaml index fdabe0ba3a..1efec57a76 100644 --- a/manifests/backup/backup-aws-s3-br.yaml +++ b/manifests/backup/backup-aws-s3-br.yaml @@ -8,6 +8,7 @@ metadata: # iam.amazonaws.com/role: "arn:aws:iam::123456789:role" spec: # backupType: full + # useKMS: false # serviceAccount: myServiceAccount br: cluster: myCluster diff --git a/manifests/backup/backup-s3-br.yaml b/manifests/backup/backup-s3-br.yaml index 4513998e49..d57c23beb7 100644 --- a/manifests/backup/backup-s3-br.yaml +++ b/manifests/backup/backup-s3-br.yaml @@ -8,6 +8,7 @@ metadata: # iam.amazonaws.com/role: "arn:aws:iam::123456789:role" spec: # backupType: full + # useKMS: false # serviceAccount: myServiceAccount br: cluster: myCluster diff --git a/manifests/backup/backup-schedule-aws-s3-br.yaml b/manifests/backup/backup-schedule-aws-s3-br.yaml index f950c6cb38..bf9501f784 100644 --- a/manifests/backup/backup-schedule-aws-s3-br.yaml +++ b/manifests/backup/backup-schedule-aws-s3-br.yaml @@ -13,6 +13,7 @@ spec: schedule: "*/2 * * * *" backupTemplate: #backupType: full + # useKMS: false # serviceAccount: myServiceAccount br: cluster: myCluster diff --git a/manifests/backup/backup-schedule-s3-br.yaml b/manifests/backup/backup-schedule-s3-br.yaml index 0fa8e2ba62..9cfda351e2 100644 --- a/manifests/backup/backup-schedule-s3-br.yaml +++ b/manifests/backup/backup-schedule-s3-br.yaml @@ -13,6 +13,7 @@ spec: schedule: "*/2 * * * *" backupTemplate: #backupType: full + # useKMS: false # serviceAccount: myServiceAccount br: cluster: myCluster diff --git a/manifests/backup/restore-aws-s3-br.yaml b/manifests/backup/restore-aws-s3-br.yaml index b9c321fc90..de5edeedb7 100644 --- a/manifests/backup/restore-aws-s3-br.yaml +++ b/manifests/backup/restore-aws-s3-br.yaml @@ -8,6 +8,7 @@ metadata: # iam.amazonaws.com/role: "arn:aws:iam::123456789:role" spec: # backupType: full + # useKMS: false # serviceAccount: myServiceAccount br: cluster: myCluster diff --git a/manifests/backup/restore-s3-br.yaml b/manifests/backup/restore-s3-br.yaml index e1ef5a8e7c..9ce686f63e 100644 --- a/manifests/backup/restore-s3-br.yaml +++ b/manifests/backup/restore-s3-br.yaml @@ -8,6 +8,7 @@ metadata: # iam.amazonaws.com/role: "arn:aws:iam::123456789:role" spec: # backupType: full + # useKMS: false # serviceAccount: myServiceAccount br: cluster: myCluster diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 5ad1b2b840..bdcc5d2fdd 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -6954,6 +6954,9 @@ spec: type: string type: object type: array + useKMS: + description: Use KMS to decrypt the secrets + type: boolean type: object type: object version: v1alpha1 @@ -7795,6 +7798,9 @@ spec: type: string type: object type: array + useKMS: + description: Use KMS to decrypt the secrets + type: boolean type: object type: object version: v1alpha1 @@ -8680,6 +8686,9 @@ spec: type: string type: object type: array + useKMS: + description: Use KMS to decrypt the secrets + type: boolean type: object maxBackups: description: MaxBackups is to specify how many backups we want to keep diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index d47a960349..d91d3d8bb5 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -772,6 +772,13 @@ func schema_pkg_apis_pingcap_v1alpha1_BackupSpec(ref common.ReferenceCallback) c Ref: ref("k8s.io/api/core/v1.Affinity"), }, }, + "useKMS": { + SchemaProps: spec.SchemaProps{ + Description: "Use KMS to decrypt the secrets", + Type: []string{"boolean"}, + Format: "", + }, + }, "serviceAccount": { SchemaProps: spec.SchemaProps{ Description: "Specify service account of backup", @@ -2850,6 +2857,13 @@ func schema_pkg_apis_pingcap_v1alpha1_RestoreSpec(ref common.ReferenceCallback) Ref: ref("k8s.io/api/core/v1.Affinity"), }, }, + "useKMS": { + SchemaProps: spec.SchemaProps{ + Description: "Use KMS to decrypt the secrets", + Type: []string{"boolean"}, + Format: "", + }, + }, "serviceAccount": { SchemaProps: spec.SchemaProps{ Description: "Specify service account of restore", diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 82363dba05..da833c1cc2 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -804,6 +804,8 @@ type BackupSpec struct { // Affinity of backup Pods // +optional Affinity *corev1.Affinity `json:"affinity,omitempty"` + // Use KMS to decrypt the secrets + UseKMS bool `json:"useKMS,omitempty"` // Specify service account of backup ServiceAccount string `json:"serviceAccount,omitempty"` } @@ -1024,6 +1026,8 @@ type RestoreSpec struct { // Affinity of restore Pods // +optional Affinity *corev1.Affinity `json:"affinity,omitempty"` + // Use KMS to decrypt the secrets + UseKMS bool `json:"useKMS,omitempty"` // Specify service account of restore ServiceAccount string `json:"serviceAccount,omitempty"` } diff --git a/pkg/backup/backup/backup_cleaner.go b/pkg/backup/backup/backup_cleaner.go index 1f48dce203..dafa6f3e38 100644 --- a/pkg/backup/backup/backup_cleaner.go +++ b/pkg/backup/backup/backup_cleaner.go @@ -128,7 +128,6 @@ func (bc *backupCleaner) makeCleanJob(backup *v1alpha1.Backup) (*batchv1.Job, st serviceAccount = backup.Spec.ServiceAccount } backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).CleanJob().Backup(name) - podSpec := &corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: backupLabel.Labels(), diff --git a/pkg/backup/backup/backup_manager.go b/pkg/backup/backup/backup_manager.go index cc13ee756f..66f33a73c5 100644 --- a/pkg/backup/backup/backup_manager.go +++ b/pkg/backup/backup/backup_manager.go @@ -163,7 +163,7 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s ns := backup.GetNamespace() name := backup.GetName() - envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.secretLister) if err != nil { return nil, reason, err } @@ -255,7 +255,7 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s ns := backup.GetNamespace() name := backup.GetName() - envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.secretLister) if err != nil { return nil, reason, err } diff --git a/pkg/backup/constants/constants.go b/pkg/backup/constants/constants.go index ba7d8d706d..ce4133c616 100644 --- a/pkg/backup/constants/constants.go +++ b/pkg/backup/constants/constants.go @@ -55,4 +55,7 @@ const ( // ServiceAccountCAPath is where is CABundle of serviceaccount locates ServiceAccountCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + + // KMS secret env prefix + KMSSecretPrefix = "KMS_ENCRYPTED" ) diff --git a/pkg/backup/restore/restore_manager.go b/pkg/backup/restore/restore_manager.go index 8a07f4a530..e29c938c9a 100644 --- a/pkg/backup/restore/restore_manager.go +++ b/pkg/backup/restore/restore_manager.go @@ -154,7 +154,7 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job ns := restore.GetNamespace() name := restore.GetName() - envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.secretLister) if err != nil { return nil, reason, err } @@ -240,7 +240,7 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo ns := restore.GetNamespace() name := restore.GetName() - envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.secretLister) if err != nil { return nil, reason, err } diff --git a/pkg/backup/util/util.go b/pkg/backup/util/util.go index 5085fc3897..4cf3a8faf3 100644 --- a/pkg/backup/util/util.go +++ b/pkg/backup/util/util.go @@ -213,8 +213,9 @@ func GenerateStorageCertEnv(ns string, provider v1alpha1.StorageProvider, secret } // GenerateTidbPasswordEnv generate the password EnvVar -func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, secretLister corelisters.SecretLister) ([]corev1.EnvVar, string, error) { +func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, useKMS bool, secretLister corelisters.SecretLister) ([]corev1.EnvVar, string, error) { var certEnv []corev1.EnvVar + var passwordKey string secret, err := secretLister.Secrets(ns).Get(tidbSecretName) if err != nil { err = fmt.Errorf("backup %s/%s get tidb secret %s failed, err: %v", ns, name, tidbSecretName, err) @@ -226,9 +227,16 @@ func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, secretLister corel err = fmt.Errorf("backup %s/%s, tidb secret %s missing password key %s", ns, name, tidbSecretName, keyStr) return certEnv, "KeyNotExist", err } + + if useKMS { + passwordKey = fmt.Sprintf("%s_%s_%s", constants.KMSSecretPrefix, constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey)) + } else { + passwordKey = fmt.Sprintf("%s_%s", constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey)) + } + certEnv = []corev1.EnvVar{ { - Name: fmt.Sprintf("%s_%s", constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey)), + Name: passwordKey, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: tidbSecretName},