From 817725d76bab9bc31fbcfae4659517670d01fb4d Mon Sep 17 00:00:00 2001 From: uriwang Date: Mon, 2 Sep 2024 17:12:40 +0800 Subject: [PATCH] =?UTF-8?q?feat(mysql):=20dbbackup=E6=94=AF=E6=8C=81rocksd?= =?UTF-8?q?b=E5=A4=87=E4=BB=BD=20#6505?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + .../db-tools/mysql-dbbackup/pkg/cst/const.go | 9 +- .../src/backupexe/dumper_physical_rocksdb.go | 214 ++++++++++++++++++ 3 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_rocksdb.go diff --git a/.gitignore b/.gitignore index 918a538bab..f6520ed87c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ pre-*-bkcodeai bkcodeai.json package-lock.json + +### PreCI ### +.codecc \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go index b57f21d68b..45912fe8f4 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go @@ -112,8 +112,9 @@ const MysqlCrondUrl = "http://127.0.0.1:9999" const MysqlRotateBinlogInstallPath = "/home/mysql/mysql-rotatebinlog" const ( - ToolMydumper = "mydumper" - ToolMysqldump = "mysqldump" - ToolXtrabackup = "xtrabackup" - ToolTmysqldump = "tmysqldump" + ToolMydumper = "mydumper" + ToolMysqldump = "mysqldump" + ToolXtrabackup = "xtrabackup" + ToolTmysqldump = "tmysqldump" + ToolMyrocksHotbackup = "myrocks_hotbackup" ) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_rocksdb.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_rocksdb.go new file mode 100644 index 0000000000..8d6fa3c225 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_rocksdb.go @@ -0,0 +1,214 @@ +package backupexe + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util" +) + +type PhysicalRocksdbDumper struct { + cfg *config.BackupConfig + dbbackupHome string + mysqlVersion string + isOfficial bool + rocksdbCmd string + storageEngine string + backupStartTime time.Time + backupEndTime time.Time +} + +func (p *PhysicalRocksdbDumper) buildArgs() []string { + args := []string{ + fmt.Sprintf("--host=%s", p.cfg.Public.MysqlHost), + fmt.Sprintf("--port=%d", p.cfg.Public.MysqlPort), + fmt.Sprintf("--user=%s", p.cfg.Public.MysqlUser), + fmt.Sprintf("--password=%s", p.cfg.Public.MysqlPasswd), + fmt.Sprintf("--checkpoint_dir=%s/MyRocks_checkpoint", p.cfg.Public.BackupDir), + fmt.Sprintf("--backup_dir=%s", p.cfg.Public.BackupDir), + "--stream=disabled", + } + + if strings.ToLower(p.cfg.Public.MysqlRole) == cst.RoleSlave { + args = append(args, "--master_info") + } + + if strings.ToLower(p.cfg.Public.MysqlRole) == cst.RoleMaster { + args = append(args, "--slave_info") + } + + return args +} + +func (p *PhysicalRocksdbDumper) initConfig(mysqlVersion string) error { + if p.cfg == nil { + return errors.New("rocksdb physical dumper config missed") + } + + cmdPath, err := os.Executable() + + if err != nil { + return err + } + + p.dbbackupHome = filepath.Dir(cmdPath) + db, err := mysqlconn.InitConn(&p.cfg.Public) + + if err != nil { + return err + } + + p.mysqlVersion, p.isOfficial = util.VersionParser(mysqlVersion) + p.storageEngine, err = mysqlconn.GetStorageEngine(db) + + if err != nil { + return err + } + p.storageEngine = strings.ToLower(p.storageEngine) + + defer func() { + _ = db.Close() + }() + + BackupTool = cst.ToolMyrocksHotbackup + return nil +} + +func (p *PhysicalRocksdbDumper) Execute(enableTimeOut bool) error { + p.backupStartTime = time.Now() + defer func() { + p.backupEndTime = time.Now() + }() + + if p.storageEngine != "rocksdb" { + err := fmt.Errorf("%s engine not support", p.storageEngine) + logger.Log.Error(err) + return err + } + + binPath := filepath.Join(p.dbbackupHome, p.rocksdbCmd) + args := p.buildArgs() + + var cmd *exec.Cmd + if enableTimeOut { + timeDiffUinx, err := GetMaxRunningTime(p.cfg.Public.BackupTimeOut) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(timeDiffUinx))*time.Second) + defer cancel() + + cmd = exec.CommandContext(ctx, "sh", "-c", fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " "))) + } else { + cmd = exec.Command("sh", "-c", fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " "))) + } + + outFile, err := os.Create( + filepath.Join( + p.dbbackupHome, + "logs", + fmt.Sprintf("rocksdb_backup_%d_%d.log", p.cfg.Public.MysqlPort, int(time.Now().Weekday())))) + + if err != nil { + logger.Log.Error("create log file failed: ", err) + return err + } + + defer func() { + _ = outFile.Close() + }() + + cmd.Stdout = outFile + cmd.Stderr = outFile + logger.Log.Info("rocksdb backup command: ", cmd.String()) + + err = cmd.Run() + if err != nil { + logger.Log.Error("run rocksdb physical backup failed: ", err) + return err + } + + return nil +} + +func (p *PhysicalRocksdbDumper) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbareport.IndexContent, error) { + db, err := mysqlconn.InitConn(&cnf.Public) + if err != nil { + return nil, errors.WithMessage(err, "IndexContent") + } + + defer func() { + _ = db.Close() + }() + + storageEngine, err := mysqlconn.GetStorageEngine(db) + if err != nil { + return nil, err + } + + storageEngine = strings.ToLower(storageEngine) + + if storageEngine != "rocksdb" { + logger.Log.Errorf("unknown storage engine(%s)", storageEngine) + return nil, nil + } + + xtrabackupBinlogInfoFileName := filepath.Join(cnf.Public.BackupDir, cnf.Public.TargetName(), "xtrabackup_binlog_info") + xtrabackupSlaveInfoFileName := filepath.Join(cnf.Public.BackupDir, cnf.Public.TargetName(), "xtrabackup_slave_info") + + tmpFileName := filepath.Join(cnf.Public.BackupDir, cnf.Public.TargetName(), "tmp_dbbackup_go.txt") + + exepath, err := os.Executable() + if err != nil { + return nil, err + } + + exepath = filepath.Dir(exepath) + qpressPath := filepath.Join(exepath, "bin", "qpress") + + var metaInfo = dbareport.IndexContent{ + BinlogInfo: dbareport.BinlogStatusInfo{}, + } + + if masterStatus, err := parseXtraBinlogInfo(qpressPath, xtrabackupBinlogInfoFileName, tmpFileName); err != nil { + return nil, err + } else { + metaInfo.BinlogInfo.ShowMasterStatus = masterStatus + metaInfo.BinlogInfo.ShowMasterStatus.MasterHost = cnf.Public.MysqlHost + metaInfo.BinlogInfo.ShowMasterStatus.MasterPort = cnf.Public.MysqlPort + } + + if mysqlRole := strings.ToLower(cnf.Public.MysqlRole); mysqlRole == cst.RoleSlave || mysqlRole == cst.RoleRepeater { + if slaveStatus, err := parseXtraSlaveInfo(qpressPath, xtrabackupSlaveInfoFileName, tmpFileName); err != nil { + return nil, err + } else { + metaInfo.BinlogInfo.ShowSlaveStatus = slaveStatus + masterHost, masterPort, err := mysqlconn.ShowMysqlSlaveStatus(db) + if err != nil { + return nil, err + } + metaInfo.BinlogInfo.ShowSlaveStatus.MasterHost = masterHost + metaInfo.BinlogInfo.ShowSlaveStatus.MasterPort = masterPort + } + } + + metaInfo.JudgeIsFullBackup(&cnf.Public) + if err = os.Remove(tmpFileName); err != nil { + return &metaInfo, err + } + + return &metaInfo, nil +}