Skip to content

Commit

Permalink
feat: Add named config overrides (sourcenetwork#659)
Browse files Browse the repository at this point in the history 
Supports a new format for configuring loggers that targets specific named loggers. It uses a comma-seperated-value (CSV) list of keyvalues.

eg:
--loglevel error,defra.cli=debug
will set the global level for all loggers to error and then the defra.cli level to debug specifically.

It supports an unbounded number of named logger values.

Additionally, introduces a new cli flag --logger which uses the same format, but for all fields of a specific logger, instead of individual fields of many loggers.

eg:
--logger defra.cli,level=debug,output=stdout,format=json
will set the level to debug, output to stdout, and format to json for the logger named defra.cli.

Co-authored-by: Andrew Sisley <a.j.sisley@hotmail.com>
  • Loading branch information
@jsimnz @AndrewSisley
jsimnz and AndrewSisley authored Jul 26, 2022
1 parent 4b18b3c commit 66ae1cd
Show file tree
Hide file tree
Showing 9 changed files with 403 additions and 87 deletions.
167 changes: 167 additions & 0 deletions cli/cli.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ import (
"encoding/json"
"fmt"
"os"
"strconv"
"strings"

"github.com/sourcenetwork/defradb/config"
"github.com/sourcenetwork/defradb/logging"
"github.com/spf13/cobra"
)

const badgerDatastoreName = "badger"
Expand Down Expand Up @@ -82,3 +84,168 @@ func hasGraphQLErrors(buf []byte) (bool, error) {
return false, nil
}
}

// parseAndConfigLog parses and then configures the given config.Config logging subconfig.
// we use log.Fatal instead of returning an error because we can't gurantee
// atomic updates, its either everything is properly set, or we Fatal()
func parseAndConfigLog(ctx context.Context, cfg *config.LoggingConfig, cmd *cobra.Command) error {
// handle --loglevels <default>,<name>=<value>,...
err := parseAndConfigLogStringParam(ctx, cfg, cfg.Level, func(l *config.LoggingConfig, v string) {
l.Level = v
})
if err != nil {
return err
}

// handle --logger <name>,<field>=<value>,...
loggerKVs, err := cmd.Flags().GetString("logger")
if err != nil {
return fmt.Errorf("can't get logger flag: %w", err)
}

if loggerKVs != "" {
if err := parseAndConfigLogAllParams(ctx, cfg, loggerKVs); err != nil {
return err
}
}
loggingConfig, err := cfg.ToLoggerConfig()
if err != nil {
return fmt.Errorf("could not get logging config: %w", err)
}
logging.SetConfig(loggingConfig)

return nil
}

func parseAndConfigLogAllParams(ctx context.Context, cfg *config.LoggingConfig, kvs string) error {
if kvs == "" {
return nil //nothing todo
}

// check if a CSV is provided
parsed := strings.Split(kvs, ",")
if len(parsed) <= 1 {
log.Fatal(ctx, "invalid --logger format, must be a csv")
}
name := parsed[0]

// verify KV format (<default>,<field>=<value>,...)
// skip the first as that will be set above
for _, kv := range parsed[1:] {
parsedKV := strings.Split(kv, "=")
if len(parsedKV) != 2 {
return fmt.Errorf("level was not provided as <key>=<value> pair: %s", kv)
}

logcfg, err := cfg.GetOrCreateNamedLogger(name)
if err != nil {
return fmt.Errorf("could not get named logger config: %w", err)
}

// handle field
switch param := strings.ToLower(parsedKV[0]); param {
case "level": // string
logcfg.Level = parsedKV[1]
case "format": // string
logcfg.Format = parsedKV[1]
case "output": // string
logcfg.OutputPath = parsedKV[1]
case "stacktrace": // bool
boolValue, err := strconv.ParseBool(parsedKV[1])
if err != nil {
return fmt.Errorf("couldn't parse kv bool: %w", err)
}
logcfg.Stacktrace = boolValue
case "nocolor": // bool
boolValue, err := strconv.ParseBool(parsedKV[1])
if err != nil {
return fmt.Errorf("couldn't parse kv bool: %w", err)
}
logcfg.NoColor = boolValue
default:
return fmt.Errorf("unknown parameter for logger: %s", param)
}
}
return nil
}

func parseAndConfigLogStringParam(
ctx context.Context,
cfg *config.LoggingConfig,
kvs string,
paramSetterFn logParamSetterStringFn) error {
if kvs == "" {
return nil //nothing todo
}

// check if a CSV is provided
// if its not a CSV, then just do the regular binding to the config
parsed := strings.Split(kvs, ",")
paramSetterFn(cfg, parsed[0])
if len(parsed) == 1 {
return nil //nothing more todo
}

// verify KV format (<default>,<name>=<value>,...)
// skip the first as that will be set above
for _, kv := range parsed[1:] {
parsedKV := strings.Split(kv, "=")
if len(parsedKV) != 2 {
return fmt.Errorf("level was not provided as <key>=<value> pair: %s", kv)
}

logcfg, err := cfg.GetOrCreateNamedLogger(parsedKV[0])
if err != nil {
return fmt.Errorf("could not get named logger config: %w", err)
}

paramSetterFn(&logcfg.LoggingConfig, parsedKV[1])
}
return nil
}

type logParamSetterStringFn func(*config.LoggingConfig, string)

//
// LEAVE FOR NOW - IMPLEMENTING SOON - PLEASE IGNORE FOR NOW
//
// func parseAndConfigLogBoolParam(
// ctx context.Context, cfg *config.LoggingConfig, kvs string, paramFn logParamSetterBoolFn) {
// if kvs == "" {
// return //nothing todo
// }

// // check if a CSV is provided
// // if its not a CSV, then just do the regular binding to the config
// parsed := strings.Split(kvs, ",")
// boolValue, err := strconv.ParseBool(parsed[0])
// if err != nil {
// log.FatalE(ctx, "couldn't parse kv bool", err)
// }
// paramFn(cfg, boolValue)
// if len(parsed) == 1 {
// return //nothing more todo
// }

// // verify KV format (<default>,<name>=<level>,...)
// // skip the first as that will be set above
// for _, kv := range parsed[1:] {
// parsedKV := strings.Split(kv, "=")
// if len(parsedKV) != 2 {
// log.Fatal(ctx, "field was not provided as <key>=<value> pair", logging.NewKV("pair", kv))
// }

// logcfg, err := cfg.GetOrCreateNamedLogger(parsedKV[0])
// if err != nil {
// log.FatalE(ctx, "could not get named logger config", err)
// }

// boolValue, err := strconv.ParseBool(parsedKV[1])
// if err != nil {
// log.FatalE(ctx, "couldn't parse kv bool", err)
// }
// paramFn(&logcfg.LoggingConfig, boolValue)
// }
// }

// type logParamSetterBoolFn func(*config.LoggingConfig, bool)
13 changes: 5 additions & 8 deletions cli/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"os"

"github.com/sourcenetwork/defradb/config"
"github.com/sourcenetwork/defradb/logging"
"github.com/spf13/cobra"
)

Expand All @@ -34,17 +33,15 @@ var initCmd = &cobra.Command{
Short: "Initialize DefraDB's root directory and configuration file",
Long: "Initialize a directory for configuration and data at the given path.",
// Load a default configuration, considering env. variables and CLI flags.
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
err := cfg.LoadWithoutRootDir()
if err != nil {
return fmt.Errorf("failed to load configuration: %w", err)
}
loggingConfig, err := cfg.GetLoggingConfig()
if err != nil {
return fmt.Errorf("failed to load logging configuration: %w", err)
}
logging.SetConfig(loggingConfig)
return nil

// parse loglevel overrides.
// binding the flags / EnvVars to the struct
return parseAndConfigLog(cmd.Context(), cfg.Log, cmd)
},
RunE: func(cmd *cobra.Command, args []string) error {
rootDirPath := ""
Expand Down
43 changes: 26 additions & 17 deletions cli/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"fmt"

"github.com/sourcenetwork/defradb/config"
"github.com/sourcenetwork/defradb/logging"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
Expand All @@ -40,27 +39,32 @@ See https://docs.source.network/BSLv0.2.txt for more information.
if err != nil {
return fmt.Errorf("failed to get root dir: %w", err)
}
defaultConfig := false
if exists {
err := cfg.Load(rootDir)
if err != nil {
return fmt.Errorf("failed to load config: %w", err)
}
loggingConfig, err := cfg.GetLoggingConfig()
if err != nil {
return fmt.Errorf("failed to get logging config: %w", err)
}
logging.SetConfig(loggingConfig)
log.Debug(cmd.Context(), fmt.Sprintf("Configuration loaded from DefraDB directory %v", rootDir))
} else {
err := cfg.LoadWithoutRootDir()
if err != nil {
return fmt.Errorf("failed to load config: %w", err)
}
loggingConfig, err := cfg.GetLoggingConfig()
if err != nil {
return fmt.Errorf("failed to get logging config: %w", err)
}
logging.SetConfig(loggingConfig)
defaultConfig = true
}

// parse loglevel overrides
// we use `cfg.Logging.Level` as an argument since the viper.Bind already handles
// binding the flags / EnvVars to the struct
if err := parseAndConfigLog(cmd.Context(), cfg.Log, cmd); err != nil {
return err
}

if defaultConfig {
log.Info(cmd.Context(), "Using default configuration")
} else {
log.Info(cmd.Context(), fmt.Sprintf("Configuration loaded from DefraDB directory %v", rootDir))

}
return nil
},
Expand All @@ -76,16 +80,21 @@ func init() {
"loglevel", cfg.Log.Level,
"Log level to use. Options are debug, info, error, fatal",
)
err := viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("loglevel"))
err := viper.BindPFlag("logging.level", rootCmd.PersistentFlags().Lookup("loglevel"))
if err != nil {
log.FatalE(context.Background(), "Could not bind log.loglevel", err)
log.FatalE(context.Background(), "Could not bind logging.loglevel", err)
}

rootCmd.PersistentFlags().String(
"logger", "",
"Named logger parameter override. usage: --logger <name>,level=<level>,output=<output>,...",
)

rootCmd.PersistentFlags().String(
"logoutput", cfg.Log.OutputPath,
"Log output path",
)
err = viper.BindPFlag("log.outputpath", rootCmd.PersistentFlags().Lookup("logoutput"))
err = viper.BindPFlag("logging.outputpath", rootCmd.PersistentFlags().Lookup("logoutput"))
if err != nil {
log.FatalE(context.Background(), "Could not bind log.outputpath", err)
}
Expand All @@ -94,7 +103,7 @@ func init() {
"logformat", cfg.Log.Format,
"Log format to use. Options are csv, json",
)
err = viper.BindPFlag("log.format", rootCmd.PersistentFlags().Lookup("logformat"))
err = viper.BindPFlag("logging.format", rootCmd.PersistentFlags().Lookup("logformat"))
if err != nil {
log.FatalE(context.Background(), "Could not bind log.format", err)
}
Expand All @@ -103,7 +112,7 @@ func init() {
"logtrace", cfg.Log.Stacktrace,
"Include stacktrace in error and fatal logs",
)
err = viper.BindPFlag("log.stacktrace", rootCmd.PersistentFlags().Lookup("logtrace"))
err = viper.BindPFlag("logging.stacktrace", rootCmd.PersistentFlags().Lookup("logtrace"))
if err != nil {
log.FatalE(context.Background(), "Could not bind log.stacktrace", err)
}
Expand Down
13 changes: 7 additions & 6 deletions cli/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,17 @@ var startCmd = &cobra.Command{
if err != nil {
return fmt.Errorf("failed to load config: %w", err)
}
loggingConfig, err := cfg.GetLoggingConfig()
if err != nil {
return fmt.Errorf("failed to get logging config: %w", err)

// parse loglevel overrides
if err := parseAndConfigLog(cmd.Context(), cfg.Log, cmd); err != nil {
return err
}
logging.SetConfig(loggingConfig)
log.Info(cmd.Context(), fmt.Sprintf("Configuration loaded from DefraDB directory %v", rootDir))
return nil
},
RunE: func(cmd *cobra.Command, _ []string) error {
log.Info(cmd.Context(), "Starting DefraDB service...")
RunE: func(cmd *cobra.Command, args []string) error {
ctx := context.Background()
log.Info(ctx, "Starting DefraDB service...")

// setup signal handlers
signalCh := make(chan os.Signal, 1)
Expand Down
Loading

0 comments on commit 66ae1cd

Please sign in to comment.