From ef3d517d284c2c8e3186fb94cce271900943d576 Mon Sep 17 00:00:00 2001 From: Jonathan Harvey-Buschel Date: Mon, 26 Aug 2024 17:04:37 -0400 Subject: [PATCH 1/2] main: add logcompressor flag --- config.go | 12 ++++++++++++ doc.go | 2 ++ log.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/config.go b/config.go index 9bbce7f69a..c5a206976a 100644 --- a/config.go +++ b/config.go @@ -41,6 +41,7 @@ const ( defaultLogLevel = "info" defaultLogDirname = "logs" defaultLogFilename = "btcd.log" + defaultLogCompressor = Gzip defaultMaxPeers = 125 defaultBanDuration = time.Hour * 24 defaultBanThreshold = 100 @@ -125,6 +126,7 @@ type config struct { FreeTxRelayLimit float64 `long:"limitfreerelay" description:"Limit relay of transactions with no transaction fee to the given amount in thousands of bytes per minute"` Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 8333, testnet: 18333)"` LogDir string `long:"logdir" description:"Directory to log output."` + LogCompressor string `long:"logcompressor" description:"Compression algorithm to use when rotating logs." choice:"gzip" choice:"zstd"` MaxOrphanTxs int `long:"maxorphantx" description:"Max number of orphan transactions to keep in memory"` MaxPeers int `long:"maxpeers" description:"Max number of inbound and outbound peers"` MiningAddrs []string `long:"miningaddr" description:"Add the specified payment address to the list of addresses to use for generated blocks -- At least one address is required if the generate option is set"` @@ -428,6 +430,7 @@ func loadConfig() (*config, []string, error) { RPCMaxConcurrentReqs: defaultMaxRPCConcurrentReqs, DataDir: defaultDataDir, LogDir: defaultLogDir, + LogCompressor: defaultLogCompressor, DbType: defaultDbType, RPCKey: defaultRPCKeyFile, RPCCert: defaultRPCCertFile, @@ -661,6 +664,15 @@ func loadConfig() (*config, []string, error) { os.Exit(0) } + // Validate that the selected log compression algorithm is supported. + if !supportedCompressor(cfg.LogCompressor) { + str := "%s: The specified log compressor [%v] is invalid" + err := fmt.Errorf(str, funcName, cfg.LogCompressor) + fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, usageMessage) + return nil, nil, err + } + // Initialize log rotation. After log rotation has been initialized, the // logger variables may be used. initLogRotator(filepath.Join(cfg.LogDir, defaultLogFilename)) diff --git a/doc.go b/doc.go index 47e4e626b7..85b9901881 100644 --- a/doc.go +++ b/doc.go @@ -76,6 +76,8 @@ Application Options: (default all interfaces port: 8333, testnet: 18333, signet: 38333) --logdir= Directory to log output + --logcompressor=[gzip,zstd] Compression algorithm to use when rotating logs. + (default: gzip) --maxorphantx= Max number of orphan transactions to keep in memory (default: 100) --maxpeers= Max number of inbound and outbound peers diff --git a/log.go b/log.go index 5707d7c23a..dcac10b350 100644 --- a/log.go +++ b/log.go @@ -105,6 +105,27 @@ var subsystemLoggers = map[string]btclog.Logger{ "TXMP": txmpLog, } +// Declare the supported compressors as exported consts for easier use from +// other projects. +const ( + Gzip = "gzip" + Zstd = "zstd" +) + +// logCompressors maps the identifier for each supported compression algorithm +// to the extension used for the compressed log files. +var logCompressors = map[string]string{ + Gzip: "gz", + Zstd: "zst", +} + +// supportedCompressor checks that the named compressor is known and supported +// for use during log rotation. +func supportedCompressor(compressor string) bool { + _, ok := logCompressors[compressor] + return ok +} + // initLogRotator initializes the logging rotater to write logs to logFile and // create roll files in the same directory. It must be called before the // package-global log rotater variables are used. @@ -121,6 +142,13 @@ func initLogRotator(logFile string) { os.Exit(1) } + // Reject unknown compressors. + if !supportedCompressor(cfg.LogCompressor) { + fmt.Fprintf(os.Stderr, "specified log compressor [%v] is "+ + "invalid", cfg.LogCompressor) + os.Exit(1) + } + logRotator = r } From ceb5bfcf335f2a040edbc956f4199f2b15fbac2d Mon Sep 17 00:00:00 2001 From: Jonathan Harvey-Buschel Date: Fri, 16 Aug 2024 14:01:20 -0400 Subject: [PATCH 2/2] log: bump logrotate dep, add zstd compressor --- config.go | 3 ++- go.mod | 3 ++- go.sum | 5 ++++- log.go | 23 +++++++++++++++++++++-- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/config.go b/config.go index c5a206976a..3f88298132 100644 --- a/config.go +++ b/config.go @@ -675,7 +675,8 @@ func loadConfig() (*config, []string, error) { // Initialize log rotation. After log rotation has been initialized, the // logger variables may be used. - initLogRotator(filepath.Join(cfg.LogDir, defaultLogFilename)) + logFile := filepath.Join(cfg.LogDir, defaultLogFilename) + initLogRotator(logFile, cfg.LogCompressor) // Parse, validate, and set debug log level(s). if err := parseAndSetDebugLevels(cfg.DebugLevel); err != nil { diff --git a/go.mod b/go.mod index 1f445d9065..adce14eef5 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/decred/dcrd/lru v1.0.0 github.com/gorilla/websocket v1.5.0 github.com/jessevdk/go-flags v1.4.0 - github.com/jrick/logrotate v1.0.0 + github.com/jrick/logrotate v1.1.2 github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 golang.org/x/crypto v0.22.0 @@ -25,6 +25,7 @@ require ( github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 // indirect + github.com/klauspost/compress v1.15.15 github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.0 // indirect golang.org/x/net v0.24.0 // indirect diff --git a/go.sum b/go.sum index bb666c89de..7afedff4a8 100644 --- a/go.sum +++ b/go.sum @@ -59,10 +59,13 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jrick/logrotate v1.1.2 h1:6ePk462NCX7TfKtNp5JJ7MbA2YIslkpfgP03TlTYMN0= +github.com/jrick/logrotate v1.1.2/go.mod h1:f9tdWggSVK3iqavGpyvegq5IhNois7KXmasU6/N96OQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/log.go b/log.go index dcac10b350..7de2f6ac69 100644 --- a/log.go +++ b/log.go @@ -6,6 +6,7 @@ package main import ( + "compress/gzip" "fmt" "os" "path/filepath" @@ -24,6 +25,7 @@ import ( "github.com/btcsuite/btclog" "github.com/jrick/logrotate/rotator" + "github.com/klauspost/compress/zstd" ) // logWriter implements an io.Writer that outputs to both standard output and @@ -128,8 +130,8 @@ func supportedCompressor(compressor string) bool { // initLogRotator initializes the logging rotater to write logs to logFile and // create roll files in the same directory. It must be called before the -// package-global log rotater variables are used. -func initLogRotator(logFile string) { +// package-global log rotator variables are used. +func initLogRotator(logFile, logCompressor string) { logDir, _ := filepath.Split(logFile) err := os.MkdirAll(logDir, 0700) if err != nil { @@ -149,6 +151,23 @@ func initLogRotator(logFile string) { os.Exit(1) } + var c rotator.Compressor + switch logCompressor { + case Gzip: + c = gzip.NewWriter(nil) + + case Zstd: + c, err = zstd.NewWriter(nil) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create zstd "+ + "compressor: %v\n", err) + os.Exit(1) + } + } + + // Apply the compressor and its file suffix to the log rotator. + r.SetCompressor(c, logCompressors[logCompressor]) + logRotator = r }