feat: zerolog based rolling log system (#535)

* feat: zerolog based rolling log system

* test: change default ParseLogLevel log level value

* fix: change default ParseLogLevel value to info

cmd/ostracon/commands/root.go

@@ -10,7 +10,6 @@ import (
 
  cfg "github.com/line/ostracon/config"
  "github.com/line/ostracon/libs/cli"
- tmflags "github.com/line/ostracon/libs/cli/flags"
  "github.com/line/ostracon/libs/log"
 )
 
@@ -61,7 +60,7 @@ var RootCmd = &cobra.Command{
  logger = log.NewOCJSONLogger(log.NewSyncWriter(os.Stdout))
  }
 
- logger, err = tmflags.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel)
+ logger, err = log.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel)
  if err != nil {
  return err
  }

config/config.go

@@ -201,6 +201,27 @@ type BaseConfig struct { //nolint: maligned
  // Output format: 'plain' (colored text) or 'json'
  LogFormat string `mapstructure:"log_format"`
 
+ // LogPath is the file to write logs to(log dir + log filename)
+ // ex) /Users/user/.app/logs/app.log
+ // If left as an empty string, log file writing is disabled.
+ LogPath string `mapstructure:"log_path"`
+
+ // LogMaxAge is the maximum number of days to retain old log files based on the
+ // timestamp encoded in their filename. Note that a day is defined as 24
+ // hours and may not exactly correspond to calendar days due to daylight
+ // savings, leap seconds, etc. The default is not to remove old log files
+ // based on age.
+ LogMaxAge int `mapstructure:"log_max_age"`
+
+ // LogMaxSize is the maximum size in megabytes of the log file before it gets
+ // rotated. It defaults to 100 megabytes.
+ LogMaxSize int `mapstructure:"log_max_size"`
+
+ // LogMaxBackups is the maximum number of old log files to retain. The default
+ // is to retain all old log files (though MaxAge may still cause them to get
+ // deleted.)
+ LogMaxBackups int `mapstructure:"log_max_backups"`
+
  // Path to the JSON file containing the initial validator set and other meta data
  Genesis string `mapstructure:"genesis_file"`
 
@@ -238,8 +259,12 @@ func DefaultBaseConfig() BaseConfig {
  Moniker: defaultMoniker,
  ProxyApp: "tcp://127.0.0.1:26658",
  ABCI: "socket",
- LogLevel: DefaultLogLevel,
+ LogLevel: DefaultPackageLogLevels(),
  LogFormat: LogFormatPlain,
+ LogPath: "",
+ LogMaxAge: 0,
+ LogMaxSize: 100,
+ LogMaxBackups: 0,
  FastSyncMode: true,
  FilterPeers: false,
  DBBackend: DefaultDBBackend,
@@ -302,6 +327,12 @@ func (cfg BaseConfig) ValidateBasic() error {
  return nil
 }
 
+// DefaultPackageLogLevels returns a default log level setting so all packages
+// log at "error", while the `state` and `main` packages log at "info"
+func DefaultPackageLogLevels() string {
+ return fmt.Sprintf("main:info,state:info,statesync:info,*:%s", DefaultLogLevel)
+}
+
 //-----------------------------------------------------------------------------
 // RPCConfig
 
@@ -1108,12 +1139,14 @@ func (cfg *ConsensusConfig) ValidateBasic() error {
  return nil
 }
 
-//-----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 // TxIndexConfig
 // Remember that Event has the following structure:
 // type: [
-// key: value,
-// ...
+//
+// key: value,
+// ...
+//
 // ]
 //
 // CompositeKeys are constructed by `type.key`

config/toml.go

@@ -122,6 +122,27 @@ log_level = "{{ .BaseConfig.LogLevel }}"
 # Output format: 'plain' (colored text) or 'json'
 log_format = "{{ .BaseConfig.LogFormat }}"
 
+# LogPath is the file to write logs to(log dir + log filename)
+# ex) /Users/user/.app/logs/app.log
+# If left as an empty string, log file writing is disabled.
+log_path = "{{ .BaseConfig.LogPath }}"
+
+# LogMaxAge is the maximum number of days to retain old log files based on the
+# timestamp encoded in their filename. Note that a day is defined as 24
+# hours and may not exactly correspond to calendar days due to daylight
+# savings, leap seconds, etc. The default is not to remove old log files
+# based on age.
+log_max_age = "{{ .BaseConfig.LogMaxAge }}"
+
+# LogMaxSize is the maximum size in megabytes of the log file before it gets
+# rotated. It defaults to 100 megabytes.
+log_max_size = "{{ .BaseConfig.LogMaxSize }}"
+
+# LogMaxBackups is the maximum number of old log files to retain. The default
+# is to retain all old log files (though MaxAge may still cause them to get
+# deleted.)
+log_max_backups = "{{ .BaseConfig.LogMaxBackups }}"
+
 ##### additional base config options #####
 
 # Path to the JSON file containing the initial validator set and other meta data

go.mod

@@ -49,6 +49,11 @@ require (
  gotest.tools v2.2.0+incompatible // indirect
 )
 
+require (
+ github.com/rs/zerolog v1.28.0
+ gopkg.in/natefinch/lumberjack.v2 v2.0.0
+)
+
 require (
  github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
  github.com/DataDog/zstd v1.4.1 // indirect
@@ -75,6 +80,8 @@ require (
  github.com/inconshreveable/mousetrap v1.0.1 // indirect
  github.com/jmhodges/levigo v1.0.0 // indirect
  github.com/magiconair/properties v1.8.6 // indirect
+ github.com/mattn/go-colorable v0.1.12 // indirect
+ github.com/mattn/go-isatty v0.0.14 // indirect
  github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
  github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 // indirect
  github.com/mitchellh/mapstructure v1.5.0 // indirect

go.sum

@@ -109,6 +109,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y=
@@ -304,6 +305,10 @@ github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QT
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -397,6 +402,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
 github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
+github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -657,6 +665,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -844,6 +853,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

libs/cli/flags/log_level.go → libs/log/log_level.go

@@ -1,11 +1,9 @@
-package flags
+package log
 
 import (
  "errors"
  "fmt"
  "strings"
-
- "github.com/line/ostracon/libs/log"
 )
 
 const (
@@ -17,8 +15,9 @@ const (
 // all other modules).
 //
 // Example:
-// ParseLogLevel("consensus:debug,mempool:debug,*:error", log.NewOCLogger(os.Stdout), "info")
-func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (log.Logger, error) {
+//
+// ParseLogLevel("consensus:debug,mempool:debug,*:error", NewOCLogger(os.Stdout), "info")
+func ParseLogLevel(lvl string, logger Logger, defaultLogLevelValue string) (Logger, error) {
  if lvl == "" {
  return nil, errors.New("empty log level")
  }
@@ -30,10 +29,10 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
  l = defaultLogLevelKey + ":" + l
  }
 
- options := make([]log.Option, 0)
+ options := make([]Option, 0)
 
  isDefaultLogLevelSet := false
- var option log.Option
+ var option Option
  var err error
 
  list := strings.Split(l, ",")
@@ -48,7 +47,7 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
  level := moduleAndLevel[1]
 
  if module == defaultLogLevelKey {
- option, err = log.AllowLevel(level)
+ option, err = AllowLevel(level)
  if err != nil {
  return nil, fmt.Errorf("failed to parse default log level (pair %s, list %s): %w", item, l, err)
  }
@@ -57,13 +56,13 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
  } else {
  switch level {
  case "debug":
- option = log.AllowDebugWith("module", module)
+ option = AllowDebugWith("module", module)
  case "info":
- option = log.AllowInfoWith("module", module)
+ option = AllowInfoWith("module", module)
  case "error":
- option = log.AllowErrorWith("module", module)
+ option = AllowErrorWith("module", module)
  case "none":
- option = log.AllowNoneWith("module", module)
+ option = AllowNoneWith("module", module)
  default:
  return nil,
  fmt.Errorf("expected either \"info\", \"debug\", \"error\" or \"none\" log level, given %s (pair %s, list %s)",
@@ -78,12 +77,12 @@ func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (
 
  // if "*" is not provided, set default global level
  if !isDefaultLogLevelSet {
- option, err = log.AllowLevel(defaultLogLevelValue)
+ option, err = AllowLevel(defaultLogLevelValue)
  if err != nil {
  return nil, err
  }
  options = append(options, option)
  }
 
- return log.NewFilter(logger, options...), nil
+ return NewFilter(logger, options...), nil
 }

libs/cli/flags/log_level_test.go → libs/log/log_level_test.go

@@ -1,11 +1,10 @@
-package flags_test
+package log_test
 
 import (
  "bytes"
  "strings"
  "testing"
 
- tmflags "github.com/line/ostracon/libs/cli/flags"
  "github.com/line/ostracon/libs/log"
 )
 
@@ -44,7 +43,7 @@ func TestParseLogLevel(t *testing.T) {
  }
 
  for _, c := range correctLogLevels {
- logger, err := tmflags.ParseLogLevel(c.lvl, jsonLogger, defaultLogLevelValue)
+ logger, err := log.ParseLogLevel(c.lvl, jsonLogger, defaultLogLevelValue)
  if err != nil {
  t.Fatal(err)
  }
@@ -87,7 +86,7 @@ func TestParseLogLevel(t *testing.T) {
 
  incorrectLogLevel := []string{"some", "mempool:some", "*:some,mempool:error"}
  for _, lvl := range incorrectLogLevel {
- if _, err := tmflags.ParseLogLevel(lvl, jsonLogger, defaultLogLevelValue); err == nil {
+ if _, err := log.ParseLogLevel(lvl, jsonLogger, defaultLogLevelValue); err == nil {
  t.Fatalf("Expected %s to produce error", lvl)
  }
  }