fix br log issue and include both 3.1 and 4.0 br in the tidb-backup-m…

…anager image

cmd/backup-manager/app/backup/backup.go

@@ -14,9 +14,11 @@
 package backup
 
 import (
+ "bufio"
  "context"
  "fmt"
  "io"
+ "io/ioutil"
  "os/exec"
  "path"
  "strings"
@@ -64,29 +66,39 @@ func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) {
  }
  fullArgs = append(fullArgs, args...)
  klog.Infof("Running br command with args: %v", fullArgs)
- cmd := exec.Command("br", fullArgs...)
- cmd.Stderr = cmd.Stdout
+ bin := "br" + backupUtil.Suffix(bo.TiKVVersion)
+ cmd := exec.Command(bin, fullArgs...)
+
  stdOut, err := cmd.StdoutPipe()
  if err != nil {
  return remotePath, fmt.Errorf("cluster %s, create stdout pipe failed, err: %v", bo, err)
  }
+ stdErr, err := cmd.StderrPipe()
+ if err != nil {
+ return remotePath, fmt.Errorf("cluster %s, create stderr pipe failed, err: %v", bo, err)
+ }
  err = cmd.Start()
  if err != nil {
  return remotePath, fmt.Errorf("cluster %s, execute br command failed, args: %s, err: %v", bo, fullArgs, err)
  }
- var tmpOutput, errMsg string
+ var errMsg string
+ reader := bufio.NewReader(stdOut)
  for {
- tmp := make([]byte, 1024)
- _, err := stdOut.Read(tmp)
- tmpOutput = string(tmp)
- if strings.Contains(tmpOutput, "[ERROR]") {
- errMsg += tmpOutput
+ line, err := reader.ReadString('\n')
+ if strings.Contains(line, "[ERROR]") {
+ errMsg += line
  }
- klog.Infof(strings.Replace(tmpOutput, "\n", "", -1))
- if err != nil {
+
+ klog.Infof(strings.Replace(line, "\n", "", -1))
+ if err != nil || io.EOF == err {
  break
  }
  }
+ tmpErr, _ := ioutil.ReadAll(stdErr)
+ if len(tmpErr) > 0 {
+ klog.Infof(string(tmpErr))
+ errMsg += string(tmpErr)
+ }
  err = cmd.Wait()
  if err != nil {
  return remotePath, fmt.Errorf("cluster %s, wait pipe message failed, errMsg %s, err: %v", bo, errMsg, err)

cmd/backup-manager/app/cmd/backup.go

@@ -42,6 +42,7 @@ func NewBackupCommand() *cobra.Command {
 
  cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace")
  cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name")
+ cmd.Flags().StringVar(&bo.TiKVVersion, "tikvVersion", util.DefaultVersion, "TiKV version")
  cmd.Flags().BoolVar(&bo.TLSClient, "client-tls", false, "Whether client tls is enabled")
  cmd.Flags().BoolVar(&bo.TLSCluster, "cluster-tls", false, "Whether cluster tls is enabled")
  return cmd

cmd/backup-manager/app/cmd/restore.go

@@ -44,6 +44,7 @@ func NewRestoreCommand() *cobra.Command {
 
  cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace")
  cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name")
+ cmd.Flags().StringVar(&ro.TiKVVersion, "tikvVersion", util.DefaultVersion, "TiKV version")
  cmd.Flags().BoolVar(&ro.TLSClient, "client-tls", false, "Whether client tls is enabled")
  cmd.Flags().BoolVar(&ro.TLSCluster, "cluster-tls", false, "Whether cluster tls is enabled")
  return cmd

cmd/backup-manager/app/restore/restore.go

@@ -14,7 +14,10 @@
 package restore
 
 import (
+ "bufio"
  "fmt"
+ "io"
+ "io/ioutil"
  "os/exec"
  "path"
  "strings"
@@ -58,29 +61,39 @@ func (ro *Options) restoreData(restore *v1alpha1.Restore) error {
  }
  fullArgs = append(fullArgs, args...)
  klog.Infof("Running br command with args: %v", fullArgs)
- cmd := exec.Command("br", fullArgs...)
- cmd.Stderr = cmd.Stdout
+ bin := "br" + backupUtil.Suffix(ro.TiKVVersion)
+ cmd := exec.Command(bin, fullArgs...)
+
  stdOut, err := cmd.StdoutPipe()
  if err != nil {
  return fmt.Errorf("cluster %s, create stdout pipe failed, err: %v", ro, err)
  }
+ stdErr, err := cmd.StderrPipe()
+ if err != nil {
+ return fmt.Errorf("cluster %s, create stderr pipe failed, err: %v", ro, err)
+ }
  err = cmd.Start()
  if err != nil {
  return fmt.Errorf("cluster %s, execute br command failed, args: %s, err: %v", ro, fullArgs, err)
  }
- var tmpOutput, errMsg string
+ var errMsg string
+ reader := bufio.NewReader(stdOut)
  for {
- tmp := make([]byte, 1024)
- _, err := stdOut.Read(tmp)
- tmpOutput = string(tmp)
- if strings.Contains(tmpOutput, "[ERROR]") {
- errMsg += tmpOutput
+ line, err := reader.ReadString('\n')
+ if strings.Contains(line, "[ERROR]") {
+ errMsg += line
  }
- klog.Infof(strings.Replace(tmpOutput, "\n", "", -1))
- if err != nil {
+ klog.Infof(strings.Replace(line, "\n", "", -1))
+ if err != nil || io.EOF == err {
  break
  }
  }
+ tmpErr, _ := ioutil.ReadAll(stdErr)
+ if len(tmpErr) > 0 {
+ klog.Infof(string(tmpErr))
+ errMsg += string(tmpErr)
+ }
+
  err = cmd.Wait()
  if err != nil {
  return fmt.Errorf("cluster %s, wait pipe message failed, errMsg %s, err: %v", ro, errMsg, err)

cmd/backup-manager/app/util/generic.go

@@ -39,6 +39,7 @@ type GenericOptions struct {
  Port int32
  Password string
  User string
+ TiKVVersion string
 }
 
 func (bo *GenericOptions) String() string {

cmd/backup-manager/app/util/util.go

@@ -19,14 +19,22 @@ import (
  "os"
  "strings"
 
+ "github.com/Masterminds/semver"
  "github.com/spf13/pflag"
+ "k8s.io/klog"
  cmdutil "k8s.io/kubectl/pkg/cmd/util"
 
  "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 )
 
 var (
- cmdHelpMsg string
+ cmdHelpMsg string
+ supportedVersions = map[string]struct{}{
+ "3.1": struct{}{},
+ "4.0": struct{}{},
+ }
+ // DefaultVersion is the default tikv and br version
+ DefaultVersion = "4.0"
 )
 
 func validCmdFlagFunc(flag *pflag.Flag) {
@@ -162,3 +170,20 @@ func constructBRGlobalOptions(config *v1alpha1.BRConfig) []string {
  }
  return args
 }
+
+// Suffix parses the major and minor version from the string and return the suffix
+func Suffix(version string) string {
+ numS := strings.Split(DefaultVersion, ".")
+ defaultSuffix := numS[0] + numS[1]
+
+ v, err := semver.NewVersion(version)
+ if err != nil {
+ klog.Errorf("Parse version %s failure, error: %v", version, err)
+ return defaultSuffix
+ }
+ parsed := fmt.Sprintf("%d.%d", v.Major(), v.Minor())
+ if _, ok := supportedVersions[parsed]; ok {
+ return fmt.Sprintf("%d%d", v.Major(), v.Minor())
+ }
+ return defaultSuffix
+}

go.mod

@@ -10,6 +10,7 @@ require (
  github.com/Azure/go-autorest/autorest/mocks v0.3.0 // indirect
  github.com/BurntSushi/toml v0.3.1
  github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect
+ github.com/Masterminds/semver v1.4.2
  github.com/Microsoft/go-winio v0.4.12 // indirect
  github.com/NYTimes/gziphandler v1.1.1 // indirect
  github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2

go.sum

@@ -51,6 +51,7 @@ github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e h1:eb0Pzkt15Bm7f2FFYv7sjY7NPFi3cPkS3tv1CcrFBWA=
 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
+github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc=

images/tidb-backup-manager/Dockerfile

@@ -2,19 +2,15 @@ FROM pingcap/tidb-enterprise-tools:latest
 ARG VERSION=v1.51.0
 ARG SHUSH_VERSION=v1.4.0
 ARG TOOLKIT_VERSION=v3.0.12
+ARG TOOLKIT_V31=v3.1.0-rc
+ARG TOOLKIT_V40=v4.0.0-rc
 RUN apk update && apk add ca-certificates
 
 RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \
- && unzip rclone-${VERSION}-linux-amd64.zip \
- && mv rclone-${VERSION}-linux-amd64/rclone /usr/local/bin \
- && chmod 755 /usr/local/bin/rclone \
- && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64
-
-RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \
- && tar -xzf br-latest-linux-amd64.tar.gz \
- && mv bin/br /usr/local/bin \
- && chmod 755 /usr/local/bin/br \
- && rm -rf br-latest-linux-amd64.tar.gz
+ && unzip rclone-${VERSION}-linux-amd64.zip \
+ && mv rclone-${VERSION}-linux-amd64/rclone /usr/local/bin \
+ && chmod 755 /usr/local/bin/rclone \
+ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64
 
 RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \
  && mv shush_linux_amd64 /usr/local/bin/shush \
@@ -29,6 +25,22 @@ RUN \
  && rm -rf tidb-toolkit-${TOOLKIT_VERSION}-linux-amd64.tar.gz \
  && rm -rf tidb-toolkit-${TOOLKIT_VERSION}-linux-amd64
 
+RUN \
+ wget -nv https://download.pingcap.org/tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \
+ && tar -xzf tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \
+ && mv tidb-toolkit-${TOOLKIT_V31}-linux-amd64/bin/br /usr/local/bin/br31 \
+ && chmod 755 /usr/local/bin/br31 \
+ && rm -rf tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \
+ && rm -rf tidb-toolkit-${TOOLKIT_V31}-linux-amd64
+
+RUN \
+ wget -nv https://download.pingcap.org/tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \
+ && tar -xzf tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \
+ && mv tidb-toolkit-${TOOLKIT_V40}-linux-amd64/bin/br /usr/local/bin/br40 \
+ && chmod 755 /usr/local/bin/br40 \
+ && rm -rf tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \
+ && rm -rf tidb-toolkit-${TOOLKIT_V40}-linux-amd64
+
 COPY bin/tidb-backup-manager /tidb-backup-manager
 COPY entrypoint.sh /entrypoint.sh
 

pkg/backup/backup/backup_manager.go

@@ -15,6 +15,7 @@ package backup
 
 import (
  "fmt"
+ "strings"
 
  "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
  "github.com/pingcap/tidb-operator/pkg/backup"
@@ -269,6 +270,13 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s
  return nil, fmt.Sprintf("failed to fetch tidbcluster %s/%s", backupNamespace, backup.Spec.BR.Cluster), err
  }
 
+ var tikvVersion string
+ tikvImage := tc.TiKVImage()
+ imageVersion := strings.Split(tikvImage, ":")
+ if len(imageVersion) == 2 {
+ tikvVersion = imageVersion[1]
+ }
+
  envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.kubeCli)
  if err != nil {
  return nil, reason, err
@@ -280,12 +288,19 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s
  }
 
  envVars = append(envVars, storageEnv...)
+ envVars = append(envVars, corev1.EnvVar{
+ Name: "BR_LOG_TO_TERM",
+ Value: string(1),
+ })
 
  args := []string{
  "backup",
  fmt.Sprintf("--namespace=%s", ns),
  fmt.Sprintf("--backupName=%s", name),
  }
+ if tikvVersion != "" {
+ args = append(args, fmt.Sprintf("--tikvVersion=%s", tikvVersion))
+ }
 
  backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).BackupJob().Backup(name)
  volumeMounts := []corev1.VolumeMount{}

pkg/backup/restore/restore_manager.go

@@ -15,6 +15,7 @@ package restore
 
 import (
  "fmt"
+ "strings"
 
  "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
  "github.com/pingcap/tidb-operator/pkg/backup"
@@ -254,6 +255,13 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo
  return nil, fmt.Sprintf("failed to fetch tidbcluster %s/%s", restoreNamespace, restore.Spec.BR.Cluster), err
  }
 
+ var tikvVersion string
+ tikvImage := tc.TiKVImage()
+ imageVersion := strings.Split(tikvImage, ":")
+ if len(imageVersion) == 2 {
+ tikvVersion = imageVersion[1]
+ }
+
  envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.kubeCli)
  if err != nil {
  return nil, reason, err
@@ -265,11 +273,18 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo
  }
 
  envVars = append(envVars, storageEnv...)
+ envVars = append(envVars, corev1.EnvVar{
+ Name: "BR_LOG_TO_TERM",
+ Value: string(1),
+ })
  args := []string{
  "restore",
  fmt.Sprintf("--namespace=%s", ns),
  fmt.Sprintf("--restoreName=%s", name),
  }
+ if tikvVersion != "" {
+ args = append(args, fmt.Sprintf("--tikvVersion=%s", tikvVersion))
+ }
 
  restoreLabel := label.NewBackup().Instance(restore.GetInstanceName()).RestoreJob().Restore(name)
  volumeMounts := []corev1.VolumeMount{}