*: add json log format support

.gitignore

@@ -3,6 +3,7 @@ bin
 .DS_Store
 .idea/
 */.idea
+.vscode/
 deps.sh
 syncer.meta
 dumped_data/*

cmd/dm-master/main.go

@@ -45,8 +45,9 @@ func main() {
 
 	// 2. init logger
 	err = log.InitLogger(&log.Config{
-		File:  cfg.LogFile,
-		Level: strings.ToLower(cfg.LogLevel),
+		File:   cfg.LogFile,
+		Level:  strings.ToLower(cfg.LogLevel),
+		Format: cfg.LogFormat,
 	})
 	if err != nil {
 		fmt.Printf("init logger error %v", errors.ErrorStack(err))

cmd/dm-syncer/config.go

@@ -47,6 +47,7 @@ type commonConfig struct {
 
 	LogLevel  string
 	LogFile   string
+	LogFormat string
 	LogRotate string
 
 	EnableGTID bool
@@ -71,6 +72,7 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask
 		Meta:             c.Meta,
 		LogLevel:         c.LogLevel,
 		LogFile:          c.LogFile,
+		LogFormat:        c.LogFormat,
 		LogRotate:        c.LogRotate,
 		EnableGTID:       c.EnableGTID,
 		SafeMode:         c.SafeMode,
@@ -96,6 +98,7 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask
 	//fs.StringVar(&cfg.PersistentTableDir, "persistent-dir", "", "syncer history table structures persistent dir; set to non-empty string will choosing history table structure according to column length when constructing DML")
 	fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal")
 	fs.StringVar(&cfg.LogFile, "log-file", "", "log file path")
+	fs.StringVar(&cfg.LogFormat, "log-format", "text", `the format of the log, "text" or "json"`)
 	fs.StringVar(&cfg.LogRotate, "log-rotate", "day", "log file rotate type, hour/day")
 	fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode")
 	fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant")
@@ -160,6 +163,7 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e
 	//fs.StringVar(&cfg.PersistentTableDir, "persistent-dir", "", "syncer history table structures persistent dir; set to non-empty string will choosing history table structure according to column length when constructing DML")
 	fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal")
 	fs.StringVar(&cfg.LogFile, "log-file", "", "log file path")
+	fs.StringVar(&cfg.LogFormat, "log-format", "text", `the format of the log, "text" or "json"`)
 	fs.StringVar(&cfg.LogRotate, "log-rotate", "day", "log file rotate type, hour/day")
 	fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode")
 	fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant")
@@ -200,6 +204,7 @@ func newCommonConfig() *commonConfig {
 	//fs.StringVar(&cfg.PersistentTableDir, "persistent-dir", "", "syncer history table structures persistent dir; set to non-empty string will choosing history table structure according to column length when constructing DML")
 	fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal")
 	fs.StringVar(&cfg.LogFile, "log-file", "", "log file path")
+	fs.StringVar(&cfg.LogFormat, "log-format", "text", `the format of the log, "text" or "json"`)
 	fs.StringVar(&cfg.LogRotate, "log-rotate", "day", "log file rotate type, hour/day")
 	fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode")
 	fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant")
@@ -218,6 +223,7 @@ type syncerConfig struct {
 	Name      string `toml:"name" json:"name"`
 	LogLevel  string `toml:"log-level" json:"log-level"`
 	LogFile   string `toml:"log-file" json:"log-file"`
+	LogFormat string `toml:"log-format" json:"log-format"`
 	LogRotate string `toml:"log-rotate" json:"log-rotate"`
 
 	StatusAddr string `toml:"status-addr" json:"status-addr"`
@@ -321,6 +327,7 @@ func (oc *syncerConfig) convertToNewFormat() (*config.SubTaskConfig, error) {
 
 		LogLevel:  oc.LogLevel,
 		LogFile:   oc.LogFile,
+		LogFormat: oc.LogFormat,
 		LogRotate: oc.LogRotate,
 
 		StatusAddr: oc.StatusAddr,

cmd/dm-syncer/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pingcap/dm/syncer"
 
 	"github.com/pingcap/errors"
+	globalLog "github.com/pingcap/log"
 	"go.uber.org/zap"
 )
 
@@ -50,14 +51,22 @@ func main() {
 
 	// 2. init logger
 	err = log.InitLogger(&log.Config{
-		File:  conf.LogFile,
-		Level: strings.ToLower(conf.LogLevel),
+		File:   conf.LogFile,
+		Format: conf.LogFormat,
+		Level:  strings.ToLower(conf.LogLevel),
 	})
 	if err != nil {
 		fmt.Printf("init logger error %v", errors.ErrorStack(err))
 		os.Exit(2)
 	}
 
+	// currently only schema tracker use global logger(std logger), simply replace it with `error` level
+	// may be we should support config logger in mock tidb later
+	confG := &globalLog.Config{Level: "error", File: globalLog.FileLogConfig{}, Format: conf.LogFormat}
+	lg, r, _ := globalLog.InitLogger(confG)
+	lg = lg.With(zap.String("component", "ddl tracker"))
+	globalLog.ReplaceGlobals(lg, r)
+
 	// 3. print process version information
 	utils.PrintInfo("dm-syncer", func() {
 		log.L().Info("", zap.Stringer("dm-syncer conf", conf))

cmd/dm-tracer/main.go

@@ -41,8 +41,9 @@ func main() {
 	}
 
 	err = log.InitLogger(&log.Config{
-		File:  cfg.LogFile,
-		Level: strings.ToLower(cfg.LogLevel),
+		File:   cfg.LogFile,
+		Level:  strings.ToLower(cfg.LogLevel),
+		Format: cfg.LogFormat,
 	})
 	if err != nil {
 		fmt.Printf("init logger error %v", errors.ErrorStack(err))

cmd/dm-worker/main.go

@@ -43,8 +43,9 @@ func main() {
 	}
 
 	err = log.InitLogger(&log.Config{
-		File:  cfg.LogFile,
-		Level: strings.ToLower(cfg.LogLevel),
+		File:   cfg.LogFile,
+		Format: cfg.LogFormat,
+		Level:  strings.ToLower(cfg.LogLevel),
 	})
 	if err != nil {
 		fmt.Printf("init logger error %v", errors.ErrorStack(err))

debug-tools/binlog-event-blackhole/config.go

@@ -23,8 +23,9 @@ import (
 type config struct {
 	*flag.FlagSet
 
-	logLevel string
-	logFile  string
+	logLevel  string
+	logFile   string
+	logFormat string
 
 	mode int
 
@@ -45,6 +46,7 @@ func newConfig() *config {
 
 	fs.StringVar(&cfg.logLevel, "L", "info", "log level: debug, info, warn, error, fatal")
 	fs.StringVar(&cfg.logFile, "log-file", "", "log file path")
+	fs.StringVar(&cfg.logFormat, "log-format", "text", `the format of the log, "text" or "json"`)
 
 	fs.IntVar(&cfg.mode, "mode", 0, "event read mode.\n1: read packet with go-mysql;\n2: read packet without go-mysql;\n3: read binary data but do nothing")
 

debug-tools/binlog-event-blackhole/main.go

@@ -42,8 +42,9 @@ func main() {
 	}
 
 	err = log.InitLogger(&log.Config{
-		File:  cfg.logFile,
-		Level: strings.ToLower(cfg.logLevel),
+		File:   cfg.logFile,
+		Level:  strings.ToLower(cfg.logLevel),
+		Format: cfg.logFormat,
 	})
 	if err != nil {
 		fmt.Printf("init logger error %v", errors.ErrorStack(err))

dm/config/subtask.go

@@ -179,6 +179,7 @@ type SubTaskConfig struct {
 	// compatible with standalone dm unit
 	LogLevel  string `toml:"log-level" json:"log-level"`
 	LogFile   string `toml:"log-file" json:"log-file"`
+	LogFormat string `toml:"log-format" json:"log-format"`
 	LogRotate string `toml:"log-rotate" json:"log-rotate"`
 
 	PprofAddr  string `toml:"pprof-addr" json:"pprof-addr"`

dm/master/config.go

@@ -65,6 +65,7 @@ func NewConfig() *Config {
 	fs.StringVar(&cfg.AdvertiseAddr, "advertise-addr", "", `advertise address for client traffic (default "${master-addr}")`)
 	fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal")
 	fs.StringVar(&cfg.LogFile, "log-file", "", "log file path")
+	fs.StringVar(&cfg.LogFormat, "log-format", "text", `the format of the log, "text" or "json"`)
 	//fs.StringVar(&cfg.LogRotate, "log-rotate", "day", "log file rotate type, hour/day")
 
 	fs.StringVar(&cfg.Name, "name", "", "human-readable name for this DM-master member")
@@ -99,6 +100,7 @@ type Config struct {
 
 	LogLevel  string `toml:"log-level" json:"log-level"`
 	LogFile   string `toml:"log-file" json:"log-file"`
+	LogFormat string `toml:"log-format" json:"log-format"`
 	LogRotate string `toml:"log-rotate" json:"log-rotate"`
 
 	RPCTimeoutStr string        `toml:"rpc-timeout" json:"rpc-timeout"`

dm/tracer/config.go

@@ -37,6 +37,7 @@ func NewConfig() *Config {
 	fs.StringVar(&cfg.TracerAddr, "tracer-addr", ":8263", "tracer API server and status addr")
 	fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal")
 	fs.StringVar(&cfg.LogFile, "log-file", "log/dm-tracer.log", "log file path")
+	fs.StringVar(&cfg.LogFormat, "log-format", "text", `the format of the log, "text" or "json"`)
 
 	return cfg
 }
@@ -47,6 +48,7 @@ type Config struct {
 
 	LogLevel   string `toml:"log-level" json:"log-level"`
 	LogFile    string `toml:"log-file" json:"log-file"`
+	LogFormat  string `toml:"log-format" json:"log-format"`
 	TracerAddr string `toml:"tracer-addr" json:"tracer-addr"`
 	Enable     bool   `toml:"enable" json:"enable"`
 	Checksum   bool   `toml:"checksum" json:"checksum"`

dm/worker/config.go

@@ -51,6 +51,7 @@ func NewConfig() *Config {
 	fs.StringVar(&cfg.AdvertiseAddr, "advertise-addr", "", `advertise address for client traffic (default "${worker-addr}")`)
 	fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal")
 	fs.StringVar(&cfg.LogFile, "log-file", "", "log file path")
+	fs.StringVar(&cfg.LogFormat, "log-format", "text", `the format of the log, "text" or "json"`)
 	//fs.StringVar(&cfg.LogRotate, "log-rotate", "day", "log file rotate type, hour/day")
 	// NOTE: add `advertise-addr` for dm-master if needed.
 	fs.StringVar(&cfg.Join, "join", "", `join to an existing cluster (usage: dm-master cluster's "${master-addr}")`)
@@ -66,6 +67,7 @@ type Config struct {
 
 	LogLevel  string `toml:"log-level" json:"log-level"`
 	LogFile   string `toml:"log-file" json:"log-file"`
+	LogFormat string `toml:"log-format" json:"log-format"`
 	LogRotate string `toml:"log-rotate" json:"log-rotate"`
 
 	Join          string `toml:"join" json:"join" `

dm/worker/server.go

@@ -408,6 +408,7 @@ func (s *Server) StartSubTask(ctx context.Context, req *pb.StartSubTaskRequest)
 
 	cfg.LogLevel = s.cfg.LogLevel
 	cfg.LogFile = s.cfg.LogFile
+	cfg.LogFormat = s.cfg.LogFormat
 	w.StartSubTask(cfg)
 
 	if resp.Result {
@@ -746,6 +747,7 @@ func (s *Server) startWorker(cfg *config.SourceConfig) error {
 	for _, subTaskCfg := range subTaskCfgm {
 		subTaskCfg.LogLevel = s.cfg.LogLevel
 		subTaskCfg.LogFile = s.cfg.LogFile
+		subTaskCfg.LogFormat = s.cfg.LogFormat
 		subTaskCfgClone := subTaskCfg
 		subTaskCfgs = append(subTaskCfgs, &subTaskCfgClone)
 	}

pkg/log/log.go

@@ -38,6 +38,8 @@ const (
 type Config struct {
 	// Log level.
 	Level string `toml:"level" json:"level"`
+	// the format of the log, "text" or "json"
+	Format string `toml:"format" json:"format"`
 	// Log filename, leave empty to disable file log.
 	File string `toml:"file" json:"file"`
 	// Max size for a single file, in MB.
@@ -108,7 +110,8 @@ func InitLogger(cfg *Config) error {
 	}
 
 	logger, props, err := pclog.InitLogger(&pclog.Config{
-		Level: cfg.Level,
+		Level:  cfg.Level,
+		Format: cfg.Format,
 		File: pclog.FileLogConfig{
 			Filename:   cfg.File,
 			MaxSize:    cfg.FileMaxSize,