feat: zerolog based rolling log system (#535)

* feat: zerolog based rolling log system

* test: change default ParseLogLevel log level value

* fix: change default ParseLogLevel value to info

cmd/ostracon/commands/root.go

@@ -10,7 +10,6 @@ import (
 
 	cfg "github.com/line/ostracon/config"
 	"github.com/line/ostracon/libs/cli"
-	tmflags "github.com/line/ostracon/libs/cli/flags"
 	"github.com/line/ostracon/libs/log"
 )
 
@@ -61,7 +60,7 @@ var RootCmd = &cobra.Command{
 			logger = log.NewOCJSONLogger(log.NewSyncWriter(os.Stdout))
 		}
 
-		logger, err = tmflags.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel)
+		logger, err = log.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel)
 		if err != nil {
 			return err
 		}

config/config.go

@@ -201,6 +201,27 @@ type BaseConfig struct { //nolint: maligned
 	// Output format: 'plain' (colored text) or 'json'
 	LogFormat string `mapstructure:"log_format"`
 
+	// LogPath is the file to write logs to(log dir + log filename)
+	// ex) /Users/user/.app/logs/app.log
+	// If left as an empty string, log file writing is disabled.
+	LogPath string `mapstructure:"log_path"`
+
+	// LogMaxAge is the maximum number of days to retain old log files based on the
+	// timestamp encoded in their filename.  Note that a day is defined as 24
+	// hours and may not exactly correspond to calendar days due to daylight
+	// savings, leap seconds, etc. The default is not to remove old log files
+	// based on age.
+	LogMaxAge int `mapstructure:"log_max_age"`
+
+	// LogMaxSize is the maximum size in megabytes of the log file before it gets
+	// rotated. It defaults to 100 megabytes.
+	LogMaxSize int `mapstructure:"log_max_size"`
+
+	// LogMaxBackups is the maximum number of old log files to retain. The default
+	// is to retain all old log files (though MaxAge may still cause them to get
+	// deleted.)
+	LogMaxBackups int `mapstructure:"log_max_backups"`
+
 	// Path to the JSON file containing the initial validator set and other meta data
 	Genesis string `mapstructure:"genesis_file"`
 
@@ -238,8 +259,12 @@ func DefaultBaseConfig() BaseConfig {
 		Moniker:            defaultMoniker,
 		ProxyApp:           "tcp://127.0.0.1:26658",
 		ABCI:               "socket",
-		LogLevel:           DefaultLogLevel,
+		LogLevel:           DefaultPackageLogLevels(),
 		LogFormat:          LogFormatPlain,
+		LogPath:            "",
+		LogMaxAge:          0,
+		LogMaxSize:         100,
+		LogMaxBackups:      0,
 		FastSyncMode:       true,
 		FilterPeers:        false,
 		DBBackend:          DefaultDBBackend,
@@ -302,6 +327,12 @@ func (cfg BaseConfig) ValidateBasic() error {
 	return nil
 }
 
+// DefaultPackageLogLevels returns a default log level setting so all packages
+// log at "error", while the `state` and `main` packages log at "info"
+func DefaultPackageLogLevels() string {
+	return fmt.Sprintf("main:info,state:info,statesync:info,*:%s", DefaultLogLevel)
+}
+
 //-----------------------------------------------------------------------------
 // RPCConfig
 
@@ -1108,12 +1139,14 @@ func (cfg *ConsensusConfig) ValidateBasic() error {
 	return nil
 }
 
-//-----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 // TxIndexConfig
 // Remember that Event has the following structure:
 // type: [
-//  key: value,
-//  ...
+//
+//	key: value,
+//	...
+//
 // ]
 //
 // CompositeKeys are constructed by `type.key`

config/toml.go

@@ -122,6 +122,27 @@ log_level = "{{ .BaseConfig.LogLevel }}"
 # Output format: 'plain' (colored text) or 'json'
 log_format = "{{ .BaseConfig.LogFormat }}"
 
+# LogPath is the file to write logs to(log dir + log filename)
+# ex) /Users/user/.app/logs/app.log
+# If left as an empty string, log file writing is disabled.
+log_path = "{{ .BaseConfig.LogPath }}"
+
+# LogMaxAge is the maximum number of days to retain old log files based on the
+# timestamp encoded in their filename.  Note that a day is defined as 24
+# hours and may not exactly correspond to calendar days due to daylight
+# savings, leap seconds, etc. The default is not to remove old log files
+# based on age.
+log_max_age = "{{ .BaseConfig.LogMaxAge }}"
+
+# LogMaxSize is the maximum size in megabytes of the log file before it gets
+# rotated. It defaults to 100 megabytes.
+log_max_size = "{{ .BaseConfig.LogMaxSize }}"
+
+# LogMaxBackups is the maximum number of old log files to retain. The default
+# is to retain all old log files (though MaxAge may still cause them to get
+# deleted.)
+log_max_backups = "{{ .BaseConfig.LogMaxBackups }}"
+
 ##### additional base config options #####
 
 # Path to the JSON file containing the initial validator set and other meta data

go.mod

@@ -49,6 +49,11 @@ require (
 	gotest.tools v2.2.0+incompatible // indirect
 )
 
+require (
+	github.com/rs/zerolog v1.28.0
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+)
+
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/DataDog/zstd v1.4.1 // indirect
@@ -75,6 +80,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect

go.sum

@@ -109,6 +109,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y=
@@ -304,6 +305,10 @@ github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QT
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -397,6 +402,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
 github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
+github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -657,6 +665,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -844,6 +853,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

libs/cli/flags/log_level.go → libs/log/log_level.go

@@ -1,11 +1,9 @@
-package flags
+package log
 
 import (
 	"errors"
 	"fmt"
 	"strings"
-
-	"github.com/line/ostracon/libs/log"
 )
 
 const (
@@ -17,8 +15,9 @@ const (
 // all other modules).
 //
 // Example:
-//		ParseLogLevel("consensus:debug,mempool:debug,*:error", log.NewOCLogger(os.Stdout), "info")
-func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (log.Logger, error) {
+//
+//	ParseLogLevel("consensus:debug,mempool:debug,*:error", NewOCLogger(os.Stdout), "info")
+func ParseLogLevel(lvl string, logger Logger, defaultLogLevelValue string) (Logger, error) {
 	if lvl == "" {
 		return nil, errors.New("empty log level")
 	}
@@ -30,10 +29,10 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
 		l = defaultLogLevelKey + ":" + l
 	}
 
-	options := make([]log.Option, 0)
+	options := make([]Option, 0)
 
 	isDefaultLogLevelSet := false
-	var option log.Option
+	var option Option
 	var err error
 
 	list := strings.Split(l, ",")
@@ -48,7 +47,7 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
 		level := moduleAndLevel[1]
 
 		if module == defaultLogLevelKey {
-			option, err = log.AllowLevel(level)
+			option, err = AllowLevel(level)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse default log level (pair %s, list %s): %w", item, l, err)
 			}
@@ -57,13 +56,13 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
 		} else {
 			switch level {
 			case "debug":
-				option = log.AllowDebugWith("module", module)
+				option = AllowDebugWith("module", module)
 			case "info":
-				option = log.AllowInfoWith("module", module)
+				option = AllowInfoWith("module", module)
 			case "error":
-				option = log.AllowErrorWith("module", module)
+				option = AllowErrorWith("module", module)
 			case "none":
-				option = log.AllowNoneWith("module", module)
+				option = AllowNoneWith("module", module)
 			default:
 				return nil,
 					fmt.Errorf("expected either \"info\", \"debug\", \"error\" or \"none\" log level, given %s (pair %s, list %s)",
@@ -78,12 +77,12 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
 
 	// if "*" is not provided, set default global level
 	if !isDefaultLogLevelSet {
-		option, err = log.AllowLevel(defaultLogLevelValue)
+		option, err = AllowLevel(defaultLogLevelValue)
 		if err != nil {
 			return nil, err
 		}
 		options = append(options, option)
 	}
 
-	return log.NewFilter(logger, options...), nil
+	return NewFilter(logger, options...), nil
 }

libs/cli/flags/log_level_test.go → libs/log/log_level_test.go

@@ -1,11 +1,10 @@
-package flags_test
+package log_test
 
 import (
 	"bytes"
 	"strings"
 	"testing"
 
-	tmflags "github.com/line/ostracon/libs/cli/flags"
 	"github.com/line/ostracon/libs/log"
 )
 
@@ -44,7 +43,7 @@ func TestParseLogLevel(t *testing.T) {
 	}
 
 	for _, c := range correctLogLevels {
-		logger, err := tmflags.ParseLogLevel(c.lvl, jsonLogger, defaultLogLevelValue)
+		logger, err := log.ParseLogLevel(c.lvl, jsonLogger, defaultLogLevelValue)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -87,7 +86,7 @@ func TestParseLogLevel(t *testing.T) {
 
 	incorrectLogLevel := []string{"some", "mempool:some", "*:some,mempool:error"}
 	for _, lvl := range incorrectLogLevel {
-		if _, err := tmflags.ParseLogLevel(lvl, jsonLogger, defaultLogLevelValue); err == nil {
+		if _, err := log.ParseLogLevel(lvl, jsonLogger, defaultLogLevelValue); err == nil {
 			t.Fatalf("Expected %s to produce error", lvl)
 		}
 	}