backup: support kms decryption secret (pingcap#1908)

images/tidb-backup-manager/Dockerfile

@@ -1,5 +1,6 @@
 FROM pingcap/tidb-enterprise-tools:latest
 ARG VERSION=v1.51.0
+ARG SHUSH_VERSION=v1.4.0
 RUN apk update && apk add ca-certificates
 
 RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \
@@ -14,6 +15,10 @@ RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \
  && chmod 755 /usr/local/bin/br \
  && rm -rf br-latest-linux-amd64.tar.gz
 
+RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \
+ && mv shush_linux_amd64 /usr/local/bin/shush \
+ && chmod 755 /usr/local/bin/shush
+
 COPY bin/tidb-backup-manager /tidb-backup-manager
 COPY entrypoint.sh /entrypoint.sh
 

images/tidb-backup-manager/entrypoint.sh

@@ -51,33 +51,40 @@ else
 fi
 
 BACKUP_BIN=/tidb-backup-manager
+if [[ -n "${AWS_DEFAULT_REGION}"]]; then
+ EXEC_COMMAND="exec"
+else
+ EXEC_COMMAND="/usr/local/bin/shush exec --"
+fi
+
+cat /tmp/rclone.conf
 
 # exec command
 case "$1" in
  backup)
  shift 1
  echo "$BACKUP_BIN backup $@"
- exec $BACKUP_BIN backup "$@"
+ $EXEC_COMMAND $BACKUP_BIN backup "$@"
  ;;
  export)
  shift 1
  echo "$BACKUP_BIN export $@"
- exec $BACKUP_BIN export "$@"
+ $EXEC_COMMAND $BACKUP_BIN export "$@"
  ;;
  restore)
  shift 1
  echo "$BACKUP_BIN restore $@"
- exec $BACKUP_BIN restore "$@"
+ $EXEC_COMMAND $BACKUP_BIN restore "$@"
  ;;
  import)
  shift 1
  echo "$BACKUP_BIN import $@"
- exec $BACKUP_BIN import "$@"
+ $EXEC_COMMAND $BACKUP_BIN import "$@"
  ;;
  clean)
  shift 1
  echo "$BACKUP_BIN clean $@"
- exec $BACKUP_BIN clean "$@"
+ $EXEC_COMMAND $BACKUP_BIN clean "$@"
  ;;
  *)
  echo "Usage: $0 {backup|restore|clean}"

manifests/backup/backup-aws-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
  # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
  # backupType: full
+ # useKMS: false
  # serviceAccount: myServiceAccount
  br:
  cluster: myCluster

manifests/backup/backup-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
  # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
  # backupType: full
+ # useKMS: false
  # serviceAccount: myServiceAccount
  br:
  cluster: myCluster

manifests/backup/backup-schedule-aws-s3-br.yaml

@@ -13,6 +13,7 @@ spec:
  schedule: "*/2 * * * *"
  backupTemplate:
  #backupType: full
+ # useKMS: false
  # serviceAccount: myServiceAccount
  br:
  cluster: myCluster

manifests/backup/backup-schedule-s3-br.yaml

@@ -13,6 +13,7 @@ spec:
  schedule: "*/2 * * * *"
  backupTemplate:
  #backupType: full
+ # useKMS: false
  # serviceAccount: myServiceAccount
  br:
  cluster: myCluster

manifests/backup/restore-aws-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
  # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
  # backupType: full
+ # useKMS: false
  # serviceAccount: myServiceAccount
  br:
  cluster: myCluster

manifests/backup/restore-s3-br.yaml

@@ -8,6 +8,7 @@ metadata:
  # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
  # backupType: full
+ # useKMS: false
  # serviceAccount: myServiceAccount
  br:
  cluster: myCluster

manifests/crd.yaml

@@ -6954,6 +6954,9 @@ spec:
  type: string
  type: object
  type: array
+ useKMS:
+ description: Use KMS to decrypt the secrets
+ type: boolean
  type: object
  type: object
  version: v1alpha1
@@ -7795,6 +7798,9 @@ spec:
  type: string
  type: object
  type: array
+ useKMS:
+ description: Use KMS to decrypt the secrets
+ type: boolean
  type: object
  type: object
  version: v1alpha1
@@ -8680,6 +8686,9 @@ spec:
  type: string
  type: object
  type: array
+ useKMS:
+ description: Use KMS to decrypt the secrets
+ type: boolean
  type: object
  maxBackups:
  description: MaxBackups is to specify how many backups we want to keep

pkg/apis/pingcap/v1alpha1/types.go

@@ -804,6 +804,8 @@ type BackupSpec struct {
  // Affinity of backup Pods
  // +optional
  Affinity *corev1.Affinity `json:"affinity,omitempty"`
+ // Use KMS to decrypt the secrets
+ UseKMS bool `json:"useKMS,omitempty"`
  // Specify service account of backup
  ServiceAccount string `json:"serviceAccount,omitempty"`
 }
@@ -1024,6 +1026,8 @@ type RestoreSpec struct {
  // Affinity of restore Pods
  // +optional
  Affinity *corev1.Affinity `json:"affinity,omitempty"`
+ // Use KMS to decrypt the secrets
+ UseKMS bool `json:"useKMS,omitempty"`
  // Specify service account of restore
  ServiceAccount string `json:"serviceAccount,omitempty"`
 }

pkg/backup/backup/backup_cleaner.go

@@ -128,7 +128,6 @@ func (bc *backupCleaner) makeCleanJob(backup *v1alpha1.Backup) (*batchv1.Job, st
  serviceAccount = backup.Spec.ServiceAccount
  }
  backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).CleanJob().Backup(name)
-
  podSpec := &corev1.PodTemplateSpec{
  ObjectMeta: metav1.ObjectMeta{
  Labels: backupLabel.Labels(),

pkg/backup/backup/backup_manager.go

@@ -163,7 +163,7 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s
  ns := backup.GetNamespace()
  name := backup.GetName()
 
- envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister)
+ envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.secretLister)
  if err != nil {
  return nil, reason, err
  }
@@ -255,7 +255,7 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s
  ns := backup.GetNamespace()
  name := backup.GetName()
 
- envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister)
+ envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.secretLister)
  if err != nil {
  return nil, reason, err
  }

pkg/backup/constants/constants.go

@@ -55,4 +55,7 @@ const (
 
  // ServiceAccountCAPath is where is CABundle of serviceaccount locates
  ServiceAccountCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+ // KMS secret env prefix
+ KMSSecretPrefix = "KMS_ENCRYPTED"
 )

pkg/backup/restore/restore_manager.go

@@ -154,7 +154,7 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job
  ns := restore.GetNamespace()
  name := restore.GetName()
 
- envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister)
+ envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.secretLister)
  if err != nil {
  return nil, reason, err
  }
@@ -240,7 +240,7 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo
  ns := restore.GetNamespace()
  name := restore.GetName()
 
- envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister)
+ envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.secretLister)
  if err != nil {
  return nil, reason, err
  }

pkg/backup/util/util.go

@@ -213,8 +213,9 @@ func GenerateStorageCertEnv(ns string, provider v1alpha1.StorageProvider, secret
 }
 
 // GenerateTidbPasswordEnv generate the password EnvVar
-func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, secretLister corelisters.SecretLister) ([]corev1.EnvVar, string, error) {
+func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, useKMS bool, secretLister corelisters.SecretLister) ([]corev1.EnvVar, string, error) {
  var certEnv []corev1.EnvVar
+ var passwordKey string
  secret, err := secretLister.Secrets(ns).Get(tidbSecretName)
  if err != nil {
  err = fmt.Errorf("backup %s/%s get tidb secret %s failed, err: %v", ns, name, tidbSecretName, err)
@@ -226,9 +227,16 @@ func GenerateTidbPasswordEnv(ns, name, tidbSecretName string, secretLister corel
  err = fmt.Errorf("backup %s/%s, tidb secret %s missing password key %s", ns, name, tidbSecretName, keyStr)
  return certEnv, "KeyNotExist", err
  }
+
+ if useKMS {
+ passwordKey = fmt.Sprintf("%s_%s_%s", constants.KMSSecretPrefix, constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey))
+ } else {
+ passwordKey = fmt.Sprintf("%s_%s", constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey))
+ }
+
  certEnv = []corev1.EnvVar{
  {
- Name: fmt.Sprintf("%s_%s", constants.BackupManagerEnvVarPrefix, strings.ToUpper(constants.TidbPasswordKey)),
+ Name: passwordKey,
  ValueFrom: &corev1.EnvVarSource{
  SecretKeyRef: &corev1.SecretKeySelector{
  LocalObjectReference: corev1.LocalObjectReference{Name: tidbSecretName},