diff --git a/charts/tidb-backup/templates/backup-job.yaml b/charts/tidb-backup/templates/backup-job.yaml index 75a3d9eaffa..8338d15d0f9 100644 --- a/charts/tidb-backup/templates/backup-job.yaml +++ b/charts/tidb-backup/templates/backup-job.yaml @@ -18,6 +18,9 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: backup helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + {{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 8 }} + {{- end }} spec: containers: - name: backup diff --git a/charts/tidb-backup/templates/backup-pvc.yaml b/charts/tidb-backup/templates/backup-pvc.yaml index 154ff0b076a..290868f8afa 100644 --- a/charts/tidb-backup/templates/backup-pvc.yaml +++ b/charts/tidb-backup/templates/backup-pvc.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.mode "backup" }} +{{- if (or (eq .Values.mode "backup") (eq .Values.mode "scheduled-restore")) }} kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/name: {{ template "chart.name" . }} app.kubernetes.io/managed-by: tidb-operator app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: backup + app.kubernetes.io/component: {{ .Values.mode }} pingcap.com/backup-cluster-name: {{ .Values.clusterName }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} spec: diff --git a/charts/tidb-backup/templates/restore-job.yaml b/charts/tidb-backup/templates/restore-job.yaml index f05276e0ee5..c73591bd073 100644 --- a/charts/tidb-backup/templates/restore-job.yaml +++ b/charts/tidb-backup/templates/restore-job.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.mode "restore" }} +{{- if (or (eq .Values.mode "restore") (eq .Values.mode "scheduled-restore")) }} apiVersion: batch/v1 kind: Job metadata: @@ -16,6 +16,9 @@ spec: app.kubernetes.io/name: {{ template "chart.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: restore + {{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 8 }} + {{- end }} spec: restartPolicy: OnFailure containers: diff --git a/charts/tidb-backup/templates/scripts/_start_restore.sh.tpl b/charts/tidb-backup/templates/scripts/_start_restore.sh.tpl index b8cbcaf97d8..99e350cf0e8 100644 --- a/charts/tidb-backup/templates/scripts/_start_restore.sh.tpl +++ b/charts/tidb-backup/templates/scripts/_start_restore.sh.tpl @@ -9,7 +9,7 @@ downloader \ --cloud=gcp \ --bucket={{ .Values.gcp.bucket }} \ --srcDir=${BACKUP_NAME} \ - --destDir=${dirname} + --destDir=/data {{- end }} {{- if .Values.ceph }} @@ -18,9 +18,21 @@ downloader \ --bucket={{ .Values.ceph.bucket }} \ --endpoint={{ .Values.ceph.endpoint }} \ --srcDir=${BACKUP_NAME} \ - --destDir=${dirname} + --destDir=/data {{- end }} +count=1 +while ! mysql -u ${TIDB_USER} -h `eval echo '${'$host'}'` -P 4000 -p${TIDB_PASSWORD} -e 'select version();' +do + echo "waiting for tidb, retry ${count} times ..." + sleep 10 + if [ ${count} -ge 180 ];then + echo "30 minutes timeout" + exit 1 + fi + let "count++" +done + /loader \ -d=${dirname} \ -h=`eval echo '${'$host'}'` \ diff --git a/charts/tidb-backup/values.yaml b/charts/tidb-backup/values.yaml index 72136066738..88b07a1d45c 100644 --- a/charts/tidb-backup/values.yaml +++ b/charts/tidb-backup/values.yaml @@ -5,7 +5,7 @@ # clusterName is the TiDB cluster name that should backup from or restore to. clusterName: demo -mode: backup # backup | restore +mode: backup # backup | restore | scheduled-restore # name is the backup name name: fullbackup-{{ date "200601021504" .Release.Time }} image: @@ -13,6 +13,10 @@ image: # https://github.com/pingcap/tidb-cloud-backup backup: pingcap/tidb-cloud-backup:20190610 +# Add additional labels for backup/restore job's pod +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +extraLabels: {} + # secretName is the name of the secret which stores user and password used for backup/restore # Note: you must give the user enough privilege to do the backup and restore # you can create the secret by: diff --git a/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml b/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml index 1c84f5d2abb..6ab29aad784 100644 --- a/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml +++ b/charts/tidb-cluster/templates/scheduled-backup-cronjob.yaml @@ -13,6 +13,8 @@ spec: schedule: {{ .Values.scheduledBackup.schedule | quote }} concurrencyPolicy: Forbid suspend: {{ .Values.scheduledBackup.suspend }} + successfulJobsHistoryLimit: {{ .Values.scheduledBackup.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.scheduledBackup.failedJobsHistoryLimit }} startingDeadlineSeconds: {{ .Values.scheduledBackup.startingDeadlineSeconds }} jobTemplate: metadata: @@ -50,13 +52,29 @@ spec: readOnly: true {{- end }} env: - - name: MY_POD_NAME + - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace {{- if .Values.scheduledBackup.gcp }} - name: GOOGLE_APPLICATION_CREDENTIALS value: /gcp/credentials.json + {{- end }} + {{- if .Values.scheduledBackup.ceph }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.scheduledBackup.ceph.secretName }} + key: access_key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.scheduledBackup.ceph.secretName }} + key: secret_key {{- end }} - name: TIDB_USER valueFrom: diff --git a/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl old mode 100644 new mode 100755 index 0dcf17f495f..73f5e76e4fe --- a/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_scheduled_backup.sh.tpl @@ -2,9 +2,13 @@ set -euo pipefail host=$(getent hosts {{ template "cluster.name" . }}-tidb | head | awk '{print $1}') -dirname=/data/scheduled-backup-`date +%Y-%m-%dT%H%M%S`-${MY_POD_NAME} -echo "making dir ${dirname}" -mkdir -p ${dirname} +timestamp=$(echo ${POD_NAME}|awk -F- '{print $(NF-1)}') +## use UTC time zone to resolve timestamp, avoiding different parsing results due to different default time zones +backupName=scheduled-backup-`date -u -d @${timestamp} "+%Y%m%d-%H%M%S"` +backupPath=/data/${backupName} + +echo "making dir ${backupPath}" +mkdir -p ${backupPath} gc_life_time=`/usr/bin/mysql -h${host} -P4000 -u{{ .Values.scheduledBackup.user }} -p${TIDB_PASSWORD} -Nse "select variable_value from mysql.tidb where variable_name='tikv_gc_life_time';"` echo "Old TiKV GC life time is ${gc_life_time}" @@ -14,13 +18,14 @@ echo "Increase TiKV GC life time to 3h" /usr/bin/mysql -h${host} -P4000 -u{{ .Values.scheduledBackup.user }} -p${TIDB_PASSWORD} -Nse "select variable_name,variable_value from mysql.tidb where variable_name='tikv_gc_life_time';" /mydumper \ - --outputdir=${dirname} \ + --outputdir=${backupPath} \ --host=${host} \ --port=4000 \ - --user={{ .Values.scheduledBackup.user }} \ + --user=${TIDB_USER} \ --password=${TIDB_PASSWORD} \ --long-query-guard=3600 \ --tidb-force-priority=LOW_PRIORITY \ + --regex '^(?!(mysql\.))' \ {{ .Values.scheduledBackup.options }} echo "Reset TiKV GC life time to ${gc_life_time}" @@ -31,7 +36,7 @@ echo "Reset TiKV GC life time to ${gc_life_time}" uploader \ --cloud=gcp \ --bucket={{ .Values.scheduledBackup.gcp.bucket }} \ - --backup-dir=${dirname} + --backup-dir=${backupPath} {{- end }} {{- if .Values.scheduledBackup.ceph }} @@ -39,5 +44,5 @@ uploader \ --cloud=ceph \ --bucket={{ .Values.scheduledBackup.ceph.bucket }} \ --endpoint={{ .Values.scheduledBackup.ceph.endpoint }} \ - --backup-dir=${dirname} + --backup-dir=${backupPath} {{- end }}