fix(mysql): 修复 proxy exporter 配置渲染问题 TencentBlueKing#6878

dbm-services/common/go-pubpkg/cmutil/file.go

@@ -55,6 +55,18 @@ func IsDirectory(path string) bool {
 	return fileInfo.IsDir()
 }
 
+// IsSymLinkFile 文件是否是软连
+func IsSymLinkFile(path string) (bool, error) {
+	info, err := os.Lstat(path)
+	if err != nil {
+		return false, err
+	} else if info.Mode()&os.ModeSymlink > 0 { // 是软链
+		return true, nil
+	} else {
+		return false, nil
+	}
+}
+
 // GetFileSize get file size from os
 func GetFileSize(path string) int64 {
 	f, err := os.Stat(path)

dbm-services/common/go-pubpkg/reportlog/report.go

@@ -39,14 +39,19 @@ func defaultLoggerOpt() *LoggerOption {
 }
 
 // NewReporter init reporter for logFile path
+// reportDir maybe like '/home/mysql/dbareport/mysql/dbbackup'
 func NewReporter(reportDir, filename string, logOpt *LoggerOption) (*Reporter, error) {
 	logFilePath := filepath.Join(reportDir, filename)
 	var reporter *Reporter = &Reporter{
 		log: &log.Logger{},
 	}
+
 	if reportDir == "" {
 		return nil, errors.Errorf("invalid reportDir:%s", reportDir)
-	} else if !cmutil.IsDirectory(reportDir) {
+	} else if link, _ := cmutil.IsSymLinkFile(reportDir); link {
+		_ = os.Remove(reportDir) // /home/mysql/dbareport 不使用软连
+	}
+	if !cmutil.IsDirectory(reportDir) {
 		if err := os.MkdirAll(reportDir, 0755); err != nil {
 			return nil, errors.Wrap(err, "create report path")
 		}
@@ -59,19 +64,7 @@ func NewReporter(reportDir, filename string, logOpt *LoggerOption) (*Reporter, e
 			f.Close()
 		}
 	}
-	/*
-		statusFile := "binlog_status.log"
-		statusLogger := &lumberjack.Logger{
-			Filename:   filepath.Join(viper.GetString("report.filepath"), statusFile),
-			MaxSize:    5, // MB
-			MaxBackups: 10,
-			MaxAge:     30,   // days
-			Compress:   true, // disabled by default
-		}
-		statusReporter := new(log.Logger)
-		statusReporter.SetOutput(statusLogger)
-		reporter.Status = &logPrint{log: statusReporter}
-	*/
+
 	if logOpt == nil {
 		logOpt = defaultLoggerOpt()
 	}

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/recover_binlog.go

@@ -641,8 +641,9 @@ func (r *RecoverBinlog) PreCheck() error {
 	if err = r.buildBinlogOptions(); err != nil {
 		return err
 	}
-	err = r.checkBinlogFiles()
-	logger.Warn("check binlog files error: %s. try to get binlog file from recover dir", err.Error())
+	if err = r.checkBinlogFiles(); err != nil {
+		logger.Warn("check binlog files error: %s. try to get binlog file from recover dir", err.Error())
+	}
 	if errors.Is(err, ErrorBinlogMissing) {
 		nameParts := strings.Split(r.BinlogFiles[0], ".")
 		if binlogFiles, err := r.GetBinlogFilesFromDir(r.BinlogDir, nameParts[0]+"."); err != nil {

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql_proxy/install_mysql_proxy.go

@@ -380,6 +380,7 @@ func (i *InstallMySQLProxyComp) initOneProxyAdminAccount(port Port) (err error)
 // 回档也会调用 install_mysql，但可能不会 install_monitor，为了避免健康误报，这个 install_mysql 阶段也渲染 exporter cnf
 func (i *InstallMySQLProxyComp) CreateExporterCnf() (err error) {
 	for _, port := range i.InsPorts {
+		logger.Info("generate exporter config for %d", port)
 		err = i.createExporterCnfIns(port)
 		if err != nil {
 			logger.Error("create exporter cnf failed %s", err.Error())
@@ -396,7 +397,7 @@ func (i *InstallMySQLProxyComp) createExporterCnfIns(port int) (err error) {
 		fmt.Sprintf("exporter_%d.cnf", port),
 	)
 
-	f, err := os.OpenFile(exporterConfigPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+	f, err := os.OpenFile(exporterConfigPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
 	if err != nil {
 		logger.Error(err.Error())
 		return err

dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/dbbackup/backup_dir.go

@@ -1,10 +1,15 @@
 package dbbackup
 
 import (
-	"dbm-services/common/go-pubpkg/logger"
-	"dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil"
 	"fmt"
 	"os"
+
+	"github.com/pkg/errors"
+
+	"dbm-services/common/go-pubpkg/cmutil"
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/core/cst"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil"
 )
 
 func (c *NewDbBackupComp) InitBackupDir() (err error) {
@@ -26,5 +31,29 @@ func (c *NewDbBackupComp) InitBackupDir() (err error) {
 	if err != nil {
 		return fmt.Errorf("execute [%s] get an error:%s,%w", cmd, output, err)
 	}
+	return c.initReportDir()
+}
+
+func (c *NewDbBackupComp) initReportDir() (err error) {
+	if c.Params.UntarOnly {
+		logger.Info("untar_only=true do not need initReportDir")
+		return nil
+	}
+	// redis 会污染 /home/mysql/dbareport，建立成软连
+	if isLink, _ := cmutil.IsSymLinkFile(cst.DBAReportBase); isLink {
+		_ = os.Remove(cst.DBAReportBase)
+	}
+	reportDir := c.Params.Configs["Public"]["ReportPath"]
+	if _, err := os.Stat(reportDir); os.IsNotExist(err) {
+		err := os.MkdirAll(reportDir, 0755)
+		if err != nil {
+			return errors.WithMessagef(err, "execute [mkdir -p %s]", reportDir)
+		}
+	}
+	cmd := fmt.Sprintf("chown -R mysql.mysql %s", reportDir)
+	output, err := osutil.ExecShellCommand(false, cmd)
+	if err != nil {
+		return fmt.Errorf("execute [%s] get an error:%s,%w", cmd, output, err)
+	}
 	return
 }

dbm-services/mysql/db-tools/dbactuator/pkg/util/db_table_filter/db_filter.go

@@ -36,7 +36,6 @@ func (c *DbTableFilter) GetDbsByConn(conn *sqlx.Conn) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	var dbs []string
 	err = conn.SelectContext(context.Background(), &dbs, `SHOW DATABASES`)
 	if err != nil {

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go

@@ -12,7 +12,6 @@ package backupexe
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"os"
 	"os/exec"
@@ -21,10 +20,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
 	"github.com/spf13/cast"
 
 	"dbm-services/common/go-pubpkg/cmutil"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util/db_table_filter"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport"
@@ -37,7 +38,7 @@ type LogicalDumperMysqldump struct {
 	cnf          *config.BackupConfig
 	dbbackupHome string
 	backupInfo   dbareport.IndexContent // for mysqldump backup
-	dbConn       *sql.DB
+	dbConn       *sqlx.Conn
 }
 
 // initConfig initializes the configuration for the logical dumper[mysqldump]
@@ -60,40 +61,45 @@ func (l *LogicalDumperMysqldump) buildArgsTableFilter() (args []string, err erro
 	dbListExclude := strings.Split(l.cnf.LogicalBackup.ExcludeDatabases, ",")
 	tbListExclude := strings.Split(l.cnf.LogicalBackup.ExcludeTables, ",")
 
-	if len(l.cnf.LogicalBackup.Databases) > 0 && len(dbList) >= 2 {
-		if len(l.cnf.LogicalBackup.Tables) == 0 { // 仅有 databases
-			args = append(args, "--databases")
-			args = append(args, dbList...)
-		} else {
-			return nil, errors.Errorf("mysqldump --tables cannot be used with multi databases")
+	var dbListFiltered []string
+	if filter, err := db_table_filter.NewFilter(dbList, tbList, dbListExclude, tbListExclude); err != nil {
+		return nil, err
+	} else {
+		l.dbConn, err = mysqlconn.InitConnx(&l.cnf.Public, context.Background())
+		if err != nil {
+			return nil, err
 		}
-	} else if len(l.cnf.LogicalBackup.Databases) > 0 && len(dbList) == 1 { // 只有一个 db，可以指定 table
-		args = append(args, l.cnf.LogicalBackup.Databases)
-		args = append(args, tbList...)
-	} else if len(l.cnf.LogicalBackup.ExcludeDatabases) > 0 {
-		if len(l.cnf.LogicalBackup.ExcludeTables) == 0 {
-			logger.Log.Info("get database list from db to exclude:")
-			l.dbConn, err = mysqlconn.InitConn(&l.cnf.Public)
-			if err != nil {
-				return nil, err
-			}
-			defer func() {
-				_ = l.dbConn.Close()
-			}()
-			if allDatabases, err := mysqlconn.GetDatabases("", l.dbConn); err != nil {
-				return nil, err
-			} else {
-				dbListExclude = append(dbListExclude, "information_schema", "performance_schema")
-				for _, d := range dbListExclude {
-					allDatabases = cmutil.StringsRemove(allDatabases, d)
-				}
-				args = append(args, "--databases", strings.Join(allDatabases, " "))
-			}
-		} else {
-			return nil, errors.Errorf("mysqldump exclude is not allowed, exclude-databases=%s, exclude-tables=%s",
-				dbListExclude, tbListExclude)
+		defer func() {
+			_ = l.dbConn.Close()
+		}()
+		dbListFiltered, err = filter.GetDbsByConn(l.dbConn)
+		if err != nil {
+			return nil, errors.WithMessage(err, "get database from instance")
 		}
 	}
+	logger.Log.Infof("database filtered:%v, tables:%v, excludeTables:%v", dbListFiltered, tbList, tbListExclude)
+
+	// mysqldump 条件：
+	// databases, exclude-databases 允许模糊匹配
+	// tables 不允许模糊匹配
+	// exclude-tables 只能为空
+	// 简单说 mysqldump 支持 --database db1 db2 db3, 或者 db1 table1 table2
+	if l.cnf.LogicalBackup.Tables == "*" && l.cnf.LogicalBackup.ExcludeTables == "" {
+		args = append(args, "--databases")
+		args = append(args, dbListFiltered...)
+	} else if len(dbListFiltered) == 1 && l.cnf.LogicalBackup.ExcludeTables == "" { // 只有一个 db，可以指定 table
+		args = append(args, dbListFiltered...)
+		args = append(args, tbList...)
+	} else if strings.Contains(l.cnf.LogicalBackup.Tables, "%") {
+		return nil, errors.Errorf("mysqldump does not support table like %s", l.cnf.LogicalBackup.Tables)
+	} else if l.cnf.LogicalBackup.ExcludeTables != "" {
+		return nil, errors.Errorf("mysqldump does not support table exclude %s", l.cnf.LogicalBackup.ExcludeTables)
+	} else if len(dbListFiltered) >= 2 && len(tbList) > 0 {
+		return nil, errors.Errorf("mysqldump --tables cannot be used with multi databases %v", dbListFiltered)
+	} else {
+		return nil, errors.Errorf("mysqldump exclude not not support, exclude-databases=%s, exclude-tables=%s",
+			dbListExclude, tbListExclude)
+	}
 	return args, nil
 }
 
@@ -169,9 +175,10 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) {
 		"--skip-opt", "--create-options", "--extended-insert", "--quick",
 		"--single-transaction", "--master-data=2",
 		"--max-allowed-packet=1G", "--no-autocommit",
-		"--set-gtid-purged=off",
-		//"--default-character-set  //
+		"--hex-blob",
+		// "--set-gtid-purged=off", // 5.7 需要
 	}
+	args = append(args, "--default-character-set", l.cnf.Public.MysqlCharset)
 	if l.cnf.Public.MysqlRole == cst.RoleSlave {
 		args = append(args, []string{
 			"--dump-slave=2", // will stop slave sql_thread

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go

@@ -55,18 +55,18 @@ func NewLogReporter(reportDir string) (*ReportLogger, error) {
 	}
 	resultReport, err := reportlog.NewReporter(reportDir, "backup_result.log", &logOpt)
 	if err != nil {
-		logger.Log.Warn("fail to init resultReporter:%s", err.Error())
+		logger.Log.Warn("fail to init resultReporter:", err.Error())
 		return nil, errors.WithMessage(err, "fail to init resultReporter")
 	}
 	filesReport, err := reportlog.NewReporter(filepath.Join(reportDir, "result"), "dbareport_result.log", &logOpt)
 	if err != nil {
-		logger.Log.Warn("fail to init statusReporter:%s", err.Error())
+		logger.Log.Warn("fail to init statusReporter:", err.Error())
 		//filesReport.Disable = true
 		return nil, errors.WithMessage(err, "fail to init statusReporter")
 	}
 	statusReport, err := reportlog.NewReporter(filepath.Join(reportDir, "status"), "backup_status.log", &logOpt)
 	if err != nil {
-		logger.Log.Warn("fail to init statusReporter:%s", err.Error())
+		logger.Log.Warn("fail to init statusReporter:", err.Error())
 		//statusReport.Disable = true
 		return nil, errors.WithMessage(err, "fail to init statusReporter")
 	}

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn/conn.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
 	"github.com/spf13/cast"
 
@@ -37,6 +38,16 @@ func InitConn(cfg *config.Public) (*sql.DB, error) {
 	return db, nil
 }
 
+// InitConnx create mysql connection of sqlx
+func InitConnx(cfg *config.Public, ctx context.Context) (*sqlx.Conn, error) {
+	if db, err := InitConn(cfg); err != nil {
+		return nil, err
+	} else {
+		dbx := sqlx.NewDb(db, "mysql")
+		return dbx.Connx(ctx)
+	}
+}
+
 // GetSingleGlobalVar get single global variable
 func GetSingleGlobalVar(variableName string, dbh *sql.DB) (val string, err error) {
 	var varName, varValue string