fix br log issue and include both 3.1 and 4.0 br in the tidb-backup-manager image (#2213)

cmd/backup-manager/app/backup/backup.go

@@ -14,9 +14,11 @@
 package backup
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os/exec"
 	"path"
 	"strings"
@@ -64,29 +66,39 @@ func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) {
 	}
 	fullArgs = append(fullArgs, args...)
 	klog.Infof("Running br command with args: %v", fullArgs)
-	cmd := exec.Command("br", fullArgs...)
-	cmd.Stderr = cmd.Stdout
+	bin := "br" + backupUtil.Suffix(bo.TiKVVersion)
+	cmd := exec.Command(bin, fullArgs...)
+
 	stdOut, err := cmd.StdoutPipe()
 	if err != nil {
 		return remotePath, fmt.Errorf("cluster %s, create stdout pipe failed, err: %v", bo, err)
 	}
+	stdErr, err := cmd.StderrPipe()
+	if err != nil {
+		return remotePath, fmt.Errorf("cluster %s, create stderr pipe failed, err: %v", bo, err)
+	}
 	err = cmd.Start()
 	if err != nil {
 		return remotePath, fmt.Errorf("cluster %s, execute br command failed, args: %s, err: %v", bo, fullArgs, err)
 	}
-	var tmpOutput, errMsg string
+	var errMsg string
+	reader := bufio.NewReader(stdOut)
 	for {
-		tmp := make([]byte, 1024)
-		_, err := stdOut.Read(tmp)
-		tmpOutput = string(tmp)
-		if strings.Contains(tmpOutput, "[ERROR]") {
-			errMsg += tmpOutput
+		line, err := reader.ReadString('\n')
+		if strings.Contains(line, "[ERROR]") {
+			errMsg += line
 		}
-		klog.Infof(strings.Replace(tmpOutput, "\n", "", -1))
-		if err != nil {
+
+		klog.Infof(strings.Replace(line, "\n", "", -1))
+		if err != nil || io.EOF == err {
 			break
 		}
 	}
+	tmpErr, _ := ioutil.ReadAll(stdErr)
+	if len(tmpErr) > 0 {
+		klog.Infof(string(tmpErr))
+		errMsg += string(tmpErr)
+	}
 	err = cmd.Wait()
 	if err != nil {
 		return remotePath, fmt.Errorf("cluster %s, wait pipe message failed, errMsg %s, err: %v", bo, errMsg, err)

cmd/backup-manager/app/cmd/backup.go

@@ -42,6 +42,7 @@ func NewBackupCommand() *cobra.Command {
 
 	cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace")
 	cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name")
+	cmd.Flags().StringVar(&bo.TiKVVersion, "tikvVersion", util.DefaultVersion, "TiKV version")
 	cmd.Flags().BoolVar(&bo.TLSClient, "client-tls", false, "Whether client tls is enabled")
 	cmd.Flags().BoolVar(&bo.TLSCluster, "cluster-tls", false, "Whether cluster tls is enabled")
 	return cmd

cmd/backup-manager/app/cmd/restore.go

@@ -44,6 +44,7 @@ func NewRestoreCommand() *cobra.Command {
 
 	cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace")
 	cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name")
+	cmd.Flags().StringVar(&ro.TiKVVersion, "tikvVersion", util.DefaultVersion, "TiKV version")
 	cmd.Flags().BoolVar(&ro.TLSClient, "client-tls", false, "Whether client tls is enabled")
 	cmd.Flags().BoolVar(&ro.TLSCluster, "cluster-tls", false, "Whether cluster tls is enabled")
 	return cmd

cmd/backup-manager/app/restore/restore.go

@@ -14,7 +14,10 @@
 package restore
 
 import (
+	"bufio"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"os/exec"
 	"path"
 	"strings"
@@ -58,29 +61,39 @@ func (ro *Options) restoreData(restore *v1alpha1.Restore) error {
 	}
 	fullArgs = append(fullArgs, args...)
 	klog.Infof("Running br command with args: %v", fullArgs)
-	cmd := exec.Command("br", fullArgs...)
-	cmd.Stderr = cmd.Stdout
+	bin := "br" + backupUtil.Suffix(ro.TiKVVersion)
+	cmd := exec.Command(bin, fullArgs...)
+
 	stdOut, err := cmd.StdoutPipe()
 	if err != nil {
 		return fmt.Errorf("cluster %s, create stdout pipe failed, err: %v", ro, err)
 	}
+	stdErr, err := cmd.StderrPipe()
+	if err != nil {
+		return fmt.Errorf("cluster %s, create stderr pipe failed, err: %v", ro, err)
+	}
 	err = cmd.Start()
 	if err != nil {
 		return fmt.Errorf("cluster %s, execute br command failed, args: %s, err: %v", ro, fullArgs, err)
 	}
-	var tmpOutput, errMsg string
+	var errMsg string
+	reader := bufio.NewReader(stdOut)
 	for {
-		tmp := make([]byte, 1024)
-		_, err := stdOut.Read(tmp)
-		tmpOutput = string(tmp)
-		if strings.Contains(tmpOutput, "[ERROR]") {
-			errMsg += tmpOutput
+		line, err := reader.ReadString('\n')
+		if strings.Contains(line, "[ERROR]") {
+			errMsg += line
 		}
-		klog.Infof(strings.Replace(tmpOutput, "\n", "", -1))
-		if err != nil {
+		klog.Infof(strings.Replace(line, "\n", "", -1))
+		if err != nil || io.EOF == err {
 			break
 		}
 	}
+	tmpErr, _ := ioutil.ReadAll(stdErr)
+	if len(tmpErr) > 0 {
+		klog.Infof(string(tmpErr))
+		errMsg += string(tmpErr)
+	}
+
 	err = cmd.Wait()
 	if err != nil {
 		return fmt.Errorf("cluster %s, wait pipe message failed, errMsg %s, err: %v", ro, errMsg, err)

cmd/backup-manager/app/util/generic.go

@@ -39,6 +39,7 @@ type GenericOptions struct {
 	Port         int32
 	Password     string
 	User         string
+	TiKVVersion  string
 }
 
 func (bo *GenericOptions) String() string {

cmd/backup-manager/app/util/util.go

@@ -19,14 +19,22 @@ import (
 	"os"
 	"strings"
 
+	"github.com/Masterminds/semver"
 	"github.com/spf13/pflag"
+	"k8s.io/klog"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 )
 
 var (
-	cmdHelpMsg string
+	cmdHelpMsg        string
+	supportedVersions = map[string]struct{}{
+		"3.1": {},
+		"4.0": {},
+	}
+	// DefaultVersion is the default tikv and br version
+	DefaultVersion = "4.0"
 )
 
 func validCmdFlagFunc(flag *pflag.Flag) {
@@ -162,3 +170,20 @@ func constructBRGlobalOptions(config *v1alpha1.BRConfig) []string {
 	}
 	return args
 }
+
+// Suffix parses the major and minor version from the string and return the suffix
+func Suffix(version string) string {
+	numS := strings.Split(DefaultVersion, ".")
+	defaultSuffix := numS[0] + numS[1]
+
+	v, err := semver.NewVersion(version)
+	if err != nil {
+		klog.Errorf("Parse version %s failure, error: %v", version, err)
+		return defaultSuffix
+	}
+	parsed := fmt.Sprintf("%d.%d", v.Major(), v.Minor())
+	if _, ok := supportedVersions[parsed]; ok {
+		return fmt.Sprintf("%d%d", v.Major(), v.Minor())
+	}
+	return defaultSuffix
+}

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/Azure/go-autorest/autorest/mocks v0.3.0 // indirect
 	github.com/BurntSushi/toml v0.3.1
 	github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect
+	github.com/Masterminds/semver v1.4.2
 	github.com/Microsoft/go-winio v0.4.12 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2

go.sum

@@ -51,6 +51,7 @@ github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e h1:eb0Pzkt15Bm7f2FFYv7sjY7NPFi3cPkS3tv1CcrFBWA=
 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
+github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc=

images/tidb-backup-manager/Dockerfile

@@ -2,19 +2,15 @@ FROM pingcap/tidb-enterprise-tools:latest
 ARG VERSION=v1.51.0
 ARG SHUSH_VERSION=v1.4.0
 ARG TOOLKIT_VERSION=v3.0.12
+ARG TOOLKIT_V31=v3.1.0-rc
+ARG TOOLKIT_V40=v4.0.0-rc
 RUN apk update && apk add ca-certificates
 
 RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \
-	&& unzip rclone-${VERSION}-linux-amd64.zip \
-	&& mv rclone-${VERSION}-linux-amd64/rclone /usr/local/bin \
-	&& chmod 755 /usr/local/bin/rclone \
-	&& rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64
-
-RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \
-    && tar -xzf br-latest-linux-amd64.tar.gz \
-    && mv bin/br /usr/local/bin \
-    && chmod 755 /usr/local/bin/br \
-    && rm -rf br-latest-linux-amd64.tar.gz
+  && unzip rclone-${VERSION}-linux-amd64.zip \
+  && mv rclone-${VERSION}-linux-amd64/rclone /usr/local/bin \
+  && chmod 755 /usr/local/bin/rclone \
+  && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64
 
 RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \
   && mv shush_linux_amd64 /usr/local/bin/shush \
@@ -29,6 +25,22 @@ RUN \
   && rm -rf tidb-toolkit-${TOOLKIT_VERSION}-linux-amd64.tar.gz \
   && rm -rf tidb-toolkit-${TOOLKIT_VERSION}-linux-amd64
 
+RUN \
+  wget -nv https://download.pingcap.org/tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \
+  && tar -xzf tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \
+  && mv tidb-toolkit-${TOOLKIT_V31}-linux-amd64/bin/br /usr/local/bin/br31 \
+  && chmod 755 /usr/local/bin/br31 \
+  && rm -rf tidb-toolkit-${TOOLKIT_V31}-linux-amd64.tar.gz \
+  && rm -rf tidb-toolkit-${TOOLKIT_V31}-linux-amd64
+
+RUN \
+  wget -nv https://download.pingcap.org/tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \
+  && tar -xzf tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \
+  && mv tidb-toolkit-${TOOLKIT_V40}-linux-amd64/bin/br /usr/local/bin/br40 \
+  && chmod 755 /usr/local/bin/br40 \
+  && rm -rf tidb-toolkit-${TOOLKIT_V40}-linux-amd64.tar.gz \
+  && rm -rf tidb-toolkit-${TOOLKIT_V40}-linux-amd64
+
 COPY bin/tidb-backup-manager /tidb-backup-manager
 COPY entrypoint.sh /entrypoint.sh
 

pkg/backup/backup/backup_manager.go

@@ -15,6 +15,7 @@ package backup
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/backup"
@@ -269,6 +270,13 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s
 		return nil, fmt.Sprintf("failed to fetch tidbcluster %s/%s", backupNamespace, backup.Spec.BR.Cluster), err
 	}
 
+	var tikvVersion string
+	tikvImage := tc.TiKVImage()
+	imageVersion := strings.Split(tikvImage, ":")
+	if len(imageVersion) == 2 {
+		tikvVersion = imageVersion[1]
+	}
+
 	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, backup.Spec.UseKMS, bm.kubeCli)
 	if err != nil {
 		return nil, reason, err
@@ -280,12 +288,19 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s
 	}
 
 	envVars = append(envVars, storageEnv...)
+	envVars = append(envVars, corev1.EnvVar{
+		Name:  "BR_LOG_TO_TERM",
+		Value: string(1),
+	})
 
 	args := []string{
 		"backup",
 		fmt.Sprintf("--namespace=%s", ns),
 		fmt.Sprintf("--backupName=%s", name),
 	}
+	if tikvVersion != "" {
+		args = append(args, fmt.Sprintf("--tikvVersion=%s", tikvVersion))
+	}
 
 	backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).BackupJob().Backup(name)
 	volumeMounts := []corev1.VolumeMount{}

pkg/backup/restore/restore_manager.go

@@ -15,6 +15,7 @@ package restore
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/backup"
@@ -254,6 +255,13 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo
 		return nil, fmt.Sprintf("failed to fetch tidbcluster %s/%s", restoreNamespace, restore.Spec.BR.Cluster), err
 	}
 
+	var tikvVersion string
+	tikvImage := tc.TiKVImage()
+	imageVersion := strings.Split(tikvImage, ":")
+	if len(imageVersion) == 2 {
+		tikvVersion = imageVersion[1]
+	}
+
 	envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, restore.Spec.UseKMS, rm.kubeCli)
 	if err != nil {
 		return nil, reason, err
@@ -265,11 +273,18 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo
 	}
 
 	envVars = append(envVars, storageEnv...)
+	envVars = append(envVars, corev1.EnvVar{
+		Name:  "BR_LOG_TO_TERM",
+		Value: string(1),
+	})
 	args := []string{
 		"restore",
 		fmt.Sprintf("--namespace=%s", ns),
 		fmt.Sprintf("--restoreName=%s", name),
 	}
+	if tikvVersion != "" {
+		args = append(args, fmt.Sprintf("--tikvVersion=%s", tikvVersion))
+	}
 
 	restoreLabel := label.NewBackup().Instance(restore.GetInstanceName()).RestoreJob().Restore(name)
 	volumeMounts := []corev1.VolumeMount{}