diff --git a/changelog/17979.txt b/changelog/17979.txt new file mode 100644 index 000000000000..81a5c023c961 --- /dev/null +++ b/changelog/17979.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core: Add read support to `sys/loggers` and `sys/loggers/:name` endpoints +``` diff --git a/helper/logging/logfile.go b/helper/logging/logfile.go new file mode 100644 index 000000000000..199bf5b41270 --- /dev/null +++ b/helper/logging/logfile.go @@ -0,0 +1,56 @@ +package logging + +import ( + "os" + "path/filepath" + "strings" + "sync" +) + +type LogFile struct { + // Name of the log file + fileName string + + // Path to the log file + logPath string + + // fileInfo is the pointer to the current file being written to + fileInfo *os.File + + // acquire is the mutex utilized to ensure we have no concurrency issues + acquire sync.Mutex +} + +func NewLogFile(logPath string, fileName string) *LogFile { + return &LogFile{ + fileName: strings.TrimSpace(fileName), + logPath: strings.TrimSpace(logPath), + } +} + +// Write is used to implement io.Writer +func (l *LogFile) Write(b []byte) (n int, err error) { + l.acquire.Lock() + defer l.acquire.Unlock() + // Create a new file if we have no file to write to + if l.fileInfo == nil { + if err := l.openNew(); err != nil { + return 0, err + } + } + + return l.fileInfo.Write(b) +} + +func (l *LogFile) openNew() error { + newFilePath := filepath.Join(l.logPath, l.fileName) + + // Try to open an existing file or create a new one if it doesn't exist. + filePointer, err := os.OpenFile(newFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o640) + if err != nil { + return err + } + + l.fileInfo = filePointer + return nil +} diff --git a/helper/logging/logger.go b/helper/logging/logger.go new file mode 100644 index 000000000000..6e87fde5387b --- /dev/null +++ b/helper/logging/logger.go @@ -0,0 +1,158 @@ +package logging + +import ( + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + log "github.com/hashicorp/go-hclog" +) + +const ( + UnspecifiedFormat LogFormat = iota + StandardFormat + JSONFormat +) + +type LogFormat int + +// LogConfig should be used to supply configuration when creating a new Vault logger +type LogConfig struct { + name string + logLevel log.Level + logFormat LogFormat + logFilePath string +} + +func NewLogConfig(name string, logLevel log.Level, logFormat LogFormat, logFilePath string) LogConfig { + return LogConfig{ + name: name, + logLevel: logLevel, + logFormat: logFormat, + logFilePath: strings.TrimSpace(logFilePath), + } +} + +func (c LogConfig) IsFormatJson() bool { + return c.logFormat == JSONFormat +} + +// Stringer implementation +func (lf LogFormat) String() string { + switch lf { + case UnspecifiedFormat: + return "unspecified" + case StandardFormat: + return "standard" + case JSONFormat: + return "json" + } + + // unreachable + return "unknown" +} + +// noErrorWriter is a wrapper to suppress errors when writing to w. +type noErrorWriter struct { + w io.Writer +} + +func (w noErrorWriter) Write(p []byte) (n int, err error) { + _, _ = w.w.Write(p) + // We purposely return n == len(p) as if write was successful + return len(p), nil +} + +// Setup creates a new logger with the specified configuration and writer +func Setup(config LogConfig, w io.Writer) (log.InterceptLogger, error) { + // Validate the log level + if config.logLevel.String() == "unknown" { + return nil, fmt.Errorf("invalid log level: %v", config.logLevel) + } + + // If out is os.Stdout and Vault is being run as a Windows Service, writes will + // fail silently, which may inadvertently prevent writes to other writers. + // noErrorWriter is used as a wrapper to suppress any errors when writing to out. + writers := []io.Writer{noErrorWriter{w: w}} + + if config.logFilePath != "" { + dir, fileName := filepath.Split(config.logFilePath) + if fileName == "" { + fileName = "vault-agent.log" + } + logFile := NewLogFile(dir, fileName) + if err := logFile.openNew(); err != nil { + return nil, fmt.Errorf("failed to set up file logging: %w", err) + } + writers = append(writers, logFile) + } + + logger := log.NewInterceptLogger(&log.LoggerOptions{ + Name: config.name, + Level: config.logLevel, + Output: io.MultiWriter(writers...), + JSONFormat: config.IsFormatJson(), + }) + return logger, nil +} + +// ParseLogFormat parses the log format from the provided string. +func ParseLogFormat(format string) (LogFormat, error) { + switch strings.ToLower(strings.TrimSpace(format)) { + case "": + return UnspecifiedFormat, nil + case "standard": + return StandardFormat, nil + case "json": + return JSONFormat, nil + default: + return UnspecifiedFormat, fmt.Errorf("unknown log format: %s", format) + } +} + +// ParseLogLevel returns the hclog.Level that corresponds with the provided level string. +// This differs hclog.LevelFromString in that it supports additional level strings. +func ParseLogLevel(logLevel string) (log.Level, error) { + var result log.Level + logLevel = strings.ToLower(strings.TrimSpace(logLevel)) + + switch logLevel { + case "trace": + result = log.Trace + case "debug": + result = log.Debug + case "notice", "info", "": + result = log.Info + case "warn", "warning": + result = log.Warn + case "err", "error": + result = log.Error + default: + return -1, errors.New(fmt.Sprintf("unknown log level: %s", logLevel)) + } + + return result, nil +} + +// TranslateLoggerLevel returns the string that corresponds with logging level of the hclog.Logger. +func TranslateLoggerLevel(logger log.Logger) (string, error) { + var result string + + if logger.IsTrace() { + result = "trace" + } else if logger.IsDebug() { + result = "debug" + } else if logger.IsInfo() { + result = "info" + } else if logger.IsWarn() { + result = "warn" + } else if logger.IsError() { + result = "error" + } else { + return "", fmt.Errorf("unknown log level") + } + + return result, nil +} diff --git a/vault/core.go b/vault/core.go index 71bba67b8bff..400e8b446de6 100644 --- a/vault/core.go +++ b/vault/core.go @@ -2860,6 +2860,7 @@ func (c *Core) AddLogger(logger log.Logger) { c.allLoggers = append(c.allLoggers, logger) } +// SetLogLevel sets logging level for all tracked loggers to the level provided func (c *Core) SetLogLevel(level log.Level) { c.allLoggersLock.RLock() defer c.allLoggersLock.RUnlock() @@ -2868,17 +2869,22 @@ func (c *Core) SetLogLevel(level log.Level) { } } -func (c *Core) SetLogLevelByName(name string, level log.Level) error { +// SetLogLevelByName sets the logging level of named logger to level provided +// if it exists. Core.allLoggers is a slice and as such it is entirely possible +// that multiple entries exist for the same name. Each instance will be modified. +func (c *Core) SetLogLevelByName(name string, level log.Level) bool { c.allLoggersLock.RLock() defer c.allLoggersLock.RUnlock() + + found := false for _, logger := range c.allLoggers { if logger.Name() == name { logger.SetLevel(level) - return nil + found = true } } - return fmt.Errorf("logger %q does not exist", name) + return found } // SetConfig sets core's config object to the newly provided config. diff --git a/vault/logical_system.go b/vault/logical_system.go index dfcd1534c2cd..a65ca24f25e3 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -20,9 +20,6 @@ import ( "time" "unicode" - "github.com/hashicorp/vault/helper/versions" - "golang.org/x/crypto/sha3" - "github.com/hashicorp/errwrap" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" @@ -32,10 +29,12 @@ import ( semver "github.com/hashicorp/go-version" "github.com/hashicorp/vault/helper/hostutil" "github.com/hashicorp/vault/helper/identity" + "github.com/hashicorp/vault/helper/logging" "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/monitor" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/random" + "github.com/hashicorp/vault/helper/versions" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/jsonutil" @@ -44,6 +43,7 @@ import ( "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/sdk/version" "github.com/mitchellh/mapstructure" + "golang.org/x/crypto/sha3" ) const ( @@ -4703,28 +4703,35 @@ func (b *SystemBackend) handleVersionHistoryList(ctx context.Context, req *logic return logical.ListResponseWithInfo(respKeys, respKeyInfo), nil } -// getLogLevel returns the hclog.Level that corresponds with the provided level string. -// This differs hclog.LevelFromString in that it supports additional level strings so -// that in remains consistent with the handling found in the "vault server" command. -func getLogLevel(logLevel string) (log.Level, error) { - var level log.Level - - switch logLevel { - case "trace": - level = log.Trace - case "debug": - level = log.Debug - case "notice", "info", "": - level = log.Info - case "warn", "warning": - level = log.Warn - case "err", "error": - level = log.Error - default: - return level, fmt.Errorf("unrecognized log level %q", logLevel) +func (b *SystemBackend) handleLoggersRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + b.Core.allLoggersLock.RLock() + defer b.Core.allLoggersLock.RUnlock() + + loggers := make(map[string]interface{}) + warnings := make([]string, 0) + + for _, logger := range b.Core.allLoggers { + loggerName := logger.Name() + + // ignore base logger + if loggerName == "" { + continue + } + + logLevel, err := logging.TranslateLoggerLevel(logger) + if err != nil { + warnings = append(warnings, fmt.Sprintf("cannot translate level for %q: %s", loggerName, err.Error())) + } else { + loggers[loggerName] = logLevel + } } - return level, nil + resp := &logical.Response{ + Data: loggers, + Warnings: warnings, + } + + return resp, nil } func (b *SystemBackend) handleLoggersWrite(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { @@ -4739,7 +4746,7 @@ func (b *SystemBackend) handleLoggersWrite(ctx context.Context, req *logical.Req return logical.ErrorResponse("level is empty"), nil } - level, err := getLogLevel(logLevel) + level, err := logging.ParseLogLevel(logLevel) if err != nil { return logical.ErrorResponse(fmt.Sprintf("invalid level provided: %s", err.Error())), nil } @@ -4750,7 +4757,7 @@ func (b *SystemBackend) handleLoggersWrite(ctx context.Context, req *logical.Req } func (b *SystemBackend) handleLoggersDelete(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { - level, err := getLogLevel(b.Core.logLevel) + level, err := logging.ParseLogLevel(b.Core.logLevel) if err != nil { return logical.ErrorResponse(fmt.Sprintf("log level from config is invalid: %s", err.Error())), nil } @@ -4760,12 +4767,63 @@ func (b *SystemBackend) handleLoggersDelete(ctx context.Context, req *logical.Re return nil, nil } +func (b *SystemBackend) handleLoggersByNameRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + nameRaw, nameOk := d.GetOk("name") + if !nameOk { + return logical.ErrorResponse("name is required"), nil + } + + name := nameRaw.(string) + if name == "" { + return logical.ErrorResponse("name is empty"), nil + } + + b.Core.allLoggersLock.RLock() + defer b.Core.allLoggersLock.RUnlock() + + loggers := make(map[string]interface{}) + warnings := make([]string, 0) + + for _, logger := range b.Core.allLoggers { + loggerName := logger.Name() + + // ignore base logger + if loggerName == "" { + continue + } + + if loggerName == name { + logLevel, err := logging.TranslateLoggerLevel(logger) + + if err != nil { + warnings = append(warnings, fmt.Sprintf("cannot translate level for %q: %s", loggerName, err.Error())) + } else { + loggers[loggerName] = logLevel + } + + break + } + } + + resp := &logical.Response{ + Data: loggers, + Warnings: warnings, + } + + return resp, nil +} + func (b *SystemBackend) handleLoggersByNameWrite(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { nameRaw, nameOk := d.GetOk("name") if !nameOk { return logical.ErrorResponse("name is required"), nil } + name := nameRaw.(string) + if name == "" { + return logical.ErrorResponse("name is empty"), nil + } + logLevelRaw, logLevelOk := d.GetOk("level") if !logLevelOk { @@ -4777,14 +4835,14 @@ func (b *SystemBackend) handleLoggersByNameWrite(ctx context.Context, req *logic return logical.ErrorResponse("level is empty"), nil } - level, err := getLogLevel(logLevel) + level, err := logging.ParseLogLevel(logLevel) if err != nil { return logical.ErrorResponse(fmt.Sprintf("invalid level provided: %s", err.Error())), nil } - err = b.Core.SetLogLevelByName(nameRaw.(string), level) - if err != nil { - return logical.ErrorResponse(fmt.Sprintf("invalid params: %s", err.Error())), nil + success := b.Core.SetLogLevelByName(name, level) + if !success { + return logical.ErrorResponse(fmt.Sprintf("logger %q not found", name)), nil } return nil, nil @@ -4796,14 +4854,19 @@ func (b *SystemBackend) handleLoggersByNameDelete(ctx context.Context, req *logi return logical.ErrorResponse("name is required"), nil } - level, err := getLogLevel(b.Core.logLevel) + level, err := logging.ParseLogLevel(b.Core.logLevel) if err != nil { return logical.ErrorResponse(fmt.Sprintf("log level from config is invalid: %s", err.Error())), nil } - err = b.Core.SetLogLevelByName(nameRaw.(string), level) - if err != nil { - return logical.ErrorResponse(fmt.Sprintf("invalid params: %s", err.Error())), nil + name := nameRaw.(string) + if name == "" { + return logical.ErrorResponse("name is empty"), nil + } + + success := b.Core.SetLogLevelByName(name, level) + if !success { + return logical.ErrorResponse(fmt.Sprintf("logger %q not found", name)), nil } return nil, nil diff --git a/vault/logical_system_paths.go b/vault/logical_system_paths.go index afd4343f22dc..939de210fc65 100644 --- a/vault/logical_system_paths.go +++ b/vault/logical_system_paths.go @@ -298,6 +298,10 @@ func (b *SystemBackend) configPaths() []*framework.Path { }, }, Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handleLoggersRead, + Summary: "Read the log level for all existing loggers.", + }, logical.UpdateOperation: &framework.PathOperation{ Callback: b.handleLoggersWrite, Summary: "Modify the log level for all existing loggers.", @@ -322,6 +326,10 @@ func (b *SystemBackend) configPaths() []*framework.Path { }, }, Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handleLoggersByNameRead, + Summary: "Read the log level for a single logger.", + }, logical.UpdateOperation: &framework.PathOperation{ Callback: b.handleLoggersByNameWrite, Summary: "Modify the log level of a single logger.", diff --git a/vault/logical_system_test.go b/vault/logical_system_test.go index a0174a05e80d..74e0f402d4cb 100644 --- a/vault/logical_system_test.go +++ b/vault/logical_system_test.go @@ -4803,66 +4803,60 @@ func TestProcessLimit(t *testing.T) { } } -func validateLevel(level string, logger hclog.Logger) bool { - switch level { - case "trace": - return logger.IsTrace() - case "debug": - return logger.IsDebug() - case "notice", "info", "": - return logger.IsInfo() - case "warn", "warning": - return logger.IsWarn() - case "err", "error": - return logger.IsError() - } - - return false -} - func TestSystemBackend_Loggers(t *testing.T) { testCases := []struct { - level string - expectError bool + level string + expectedLevel string + expectError bool }{ { + "trace", "trace", false, }, { + "debug", "debug", false, }, { "notice", + "info", false, }, { + "info", "info", false, }, { + "warn", "warn", false, }, { "warning", + "warn", false, }, { "err", + "error", false, }, { + "error", "error", false, }, { "", + "info", true, }, { "invalid", + "", true, }, } @@ -4875,7 +4869,33 @@ func TestSystemBackend_Loggers(t *testing.T) { core, b, _ := testCoreSystemBackend(t) + // Test core overrides logging level outside of config, + // an initial delete will ensure that we an initial read + // to get expected values is based off of config and not + // the test override that is hidden from this test req := &logical.Request{ + Path: "loggers", + Operation: logical.DeleteOperation, + } + + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + + req = &logical.Request{ + Path: "loggers", + Operation: logical.ReadOperation, + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + + initialLoggers := resp.Data + + req = &logical.Request{ Path: "loggers", Operation: logical.UpdateOperation, Data: map[string]interface{}{ @@ -4883,7 +4903,7 @@ func TestSystemBackend_Loggers(t *testing.T) { }, } - resp, err := b.HandleRequest(namespace.RootContext(nil), req) + resp, err = b.HandleRequest(namespace.RootContext(nil), req) respIsError := resp != nil && resp.IsError() if err != nil || (!tc.expectError && respIsError) { @@ -4895,15 +4915,32 @@ func TestSystemBackend_Loggers(t *testing.T) { } if !tc.expectError { + req = &logical.Request{ + Path: "loggers", + Operation: logical.ReadOperation, + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + for _, logger := range core.allLoggers { - if !validateLevel(tc.level, logger) { - t.Fatalf("expected logger %q to be %q", logger.Name(), tc.level) + loggerName := logger.Name() + levelRaw, ok := resp.Data[loggerName] + + if !ok { + t.Errorf("logger %q not found in response", loggerName) + } + + if levelStr := levelRaw.(string); levelStr != tc.expectedLevel { + t.Errorf("unexpected level of logger %q, expected: %s, actual: %s", loggerName, tc.expectedLevel, levelStr) } } } req = &logical.Request{ - Path: fmt.Sprintf("loggers"), + Path: "loggers", Operation: logical.DeleteOperation, } @@ -4912,9 +4949,29 @@ func TestSystemBackend_Loggers(t *testing.T) { t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) } + req = &logical.Request{ + Path: "loggers", + Operation: logical.ReadOperation, + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + for _, logger := range core.allLoggers { - if !validateLevel(core.logLevel, logger) { - t.Errorf("expected level of logger %q to match original config", logger.Name()) + loggerName := logger.Name() + levelRaw, currentOk := resp.Data[loggerName] + initialLevelRaw, initialOk := initialLoggers[loggerName] + + if !currentOk || !initialOk { + t.Errorf("logger %q not found", loggerName) + } + + levelStr := levelRaw.(string) + initialLevelStr := initialLevelRaw.(string) + if levelStr != initialLevelStr { + t.Errorf("expected level of logger %q to match original config, expected: %s, actual: %s", loggerName, initialLevelStr, levelStr) } } }) @@ -4925,78 +4982,91 @@ func TestSystemBackend_LoggersByName(t *testing.T) { testCases := []struct { logger string level string + expectedLevel string expectWriteError bool expectDeleteError bool }{ { "core", "trace", + "trace", false, false, }, { "token", "debug", + "debug", false, false, }, { "audit", "notice", + "info", false, false, }, { "expiration", "info", + "info", false, false, }, { "policy", "warn", + "warn", false, false, }, { "activity", "warning", + "warn", false, false, }, { "identity", "err", + "error", false, false, }, { "rollback", "error", + "error", false, false, }, { "system", "", + "does-not-matter", true, false, }, { "quotas", "invalid", + "does-not-matter", true, false, }, { "", "info", + "does-not-matter", true, true, }, { "does_not_exist", "error", + "does-not-matter", true, true, }, @@ -5010,16 +5080,41 @@ func TestSystemBackend_LoggersByName(t *testing.T) { core, b, _ := testCoreSystemBackend(t) + // Test core overrides logging level outside of config, + // an initial delete will ensure that we an initial read + // to get expected values is based off of config and not + // the test override that is hidden from this test req := &logical.Request{ + Path: "loggers", + Operation: logical.DeleteOperation, + } + + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + + req = &logical.Request{ + Path: "loggers", + Operation: logical.ReadOperation, + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + + initialLoggers := resp.Data + + req = &logical.Request{ Path: fmt.Sprintf("loggers/%s", tc.logger), Operation: logical.UpdateOperation, Data: map[string]interface{}{ - "name": tc.logger, "level": tc.level, }, } - resp, err := b.HandleRequest(namespace.RootContext(nil), req) + resp, err = b.HandleRequest(namespace.RootContext(nil), req) respIsError := resp != nil && resp.IsError() if err != nil || (!tc.expectWriteError && respIsError) { @@ -5031,13 +5126,34 @@ func TestSystemBackend_LoggersByName(t *testing.T) { } if !tc.expectWriteError { + req = &logical.Request{ + Path: "loggers", + Operation: logical.ReadOperation, + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + for _, logger := range core.allLoggers { - if logger.Name() != tc.logger && !validateLevel(core.logLevel, logger) { - t.Errorf("expected level of logger %q to be unchanged", logger.Name()) + loggerName := logger.Name() + levelRaw, currentOk := resp.Data[loggerName] + initialLevelRaw, initialOk := initialLoggers[loggerName] + + if !currentOk || !initialOk { + t.Errorf("logger %q not found", loggerName) + } + + levelStr := levelRaw.(string) + initialLevelStr := initialLevelRaw.(string) + + if loggerName == tc.logger && levelStr != tc.expectedLevel { + t.Fatalf("expected logger %q to be %q, actual: %s", loggerName, tc.expectedLevel, levelStr) } - if !validateLevel(tc.level, logger) { - t.Fatalf("expected logger %q to be %q", logger.Name(), tc.level) + if loggerName != tc.logger && levelStr != initialLevelStr { + t.Errorf("expected level of logger %q to be unchanged, exepcted: %s, actual: %s", loggerName, initialLevelStr, levelStr) } } } @@ -5045,9 +5161,6 @@ func TestSystemBackend_LoggersByName(t *testing.T) { req = &logical.Request{ Path: fmt.Sprintf("loggers/%s", tc.logger), Operation: logical.DeleteOperation, - Data: map[string]interface{}{ - "name": tc.logger, - }, } resp, err = b.HandleRequest(namespace.RootContext(nil), req) @@ -5062,10 +5175,28 @@ func TestSystemBackend_LoggersByName(t *testing.T) { } if !tc.expectDeleteError { - for _, logger := range core.allLoggers { - if !validateLevel(core.logLevel, logger) { - t.Errorf("expected level of logger %q to match original config", logger.Name()) - } + req = &logical.Request{ + Path: fmt.Sprintf("loggers/%s", tc.logger), + Operation: logical.ReadOperation, + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("unexpected error, err: %v, resp: %#v", err, resp) + } + + currentLevel, ok := resp.Data[tc.logger].(string) + if !ok { + t.Fatalf("expected resp to include %q, resp: %#v", tc.logger, resp) + } + + initialLevel, ok := initialLoggers[tc.logger].(string) + if !ok { + t.Fatalf("expected initial loggers to include %q, resp: %#v", tc.logger, initialLoggers) + } + + if currentLevel != initialLevel { + t.Errorf("expected level of logger %q to match original config, expected: %s, actual: %s", tc.logger, initialLevel, currentLevel) } } })