Skip to content

Commit

Permalink
🛠️ new logger/zerologadapter for retryable requests (#4903)
Browse files Browse the repository at this point in the history
We use the `retryablehttp` package to retry http requests and we always
want the logs to go to `debug` level so, we need an adapter.

We have implemented this adapter many times, this change removes all the
duplicated code and creates a util package that all providers can use.

Example:

```go
retryClient := retryablehttp.NewClient()
retryClient.Logger = zerologadapter.New(log.Logger)
```

* ↪️  move package to `logger/`

---------

Signed-off-by: Salim Afiune Maya <afiune@mondoo.com>
  • Loading branch information
afiune authored Nov 21, 2024
1 parent a90bf3a commit 55dbd36
Show file tree
Hide file tree
Showing 7 changed files with 210 additions and 126 deletions.
57 changes: 57 additions & 0 deletions logger/zerologadapter/adapter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

package zerologadapter

import "github.com/rs/zerolog"

// New returns a new adapter for the zerolog logger to the LeveledLogger interface. This struct is
// mainly used in conjunction with a retryable http client to convert all retry logs to debug logs.
//
// NOTE that all messages will go to debug level.
//
// e.g.
// ```go
// retryClient := retryablehttp.NewClient()
// retryClient.Logger = zerologadapter.New(log.Logger)
// ```
func New(logger zerolog.Logger) *Adapter {
return &Adapter{logger}
}

type Adapter struct {
logger zerolog.Logger
}

func (z *Adapter) Msg(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Error(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Info(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Debug(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Warn(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func convertToFields(keysAndValues ...interface{}) map[string]interface{} {
fields := make(map[string]interface{})
for i := 0; i < len(keysAndValues); i += 2 {
if i+1 < len(keysAndValues) {
keyString, ok := keysAndValues[i].(string)
if ok { // safety first, eventhough we always expect a string
fields[keyString] = keysAndValues[i+1]
}
}
}
return fields
}
61 changes: 61 additions & 0 deletions logger/zerologadapter/adapter_internal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

package zerologadapter

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestConvertToFields(t *testing.T) {
t.Run("Valid key-value pairs", func(t *testing.T) {
input := []interface{}{"key1", "value1", "key2", 42, "key3", true}
expected := map[string]interface{}{
"key1": "value1",
"key2": 42,
"key3": true,
}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Odd number of elements", func(t *testing.T) {
input := []interface{}{"key1", "value1", "key2"}
expected := map[string]interface{}{
"key1": "value1",
}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Non-string keys are ignored", func(t *testing.T) {
input := []interface{}{123, "value1", "key2", 42, 3.14, "value3", "key3", true}
expected := map[string]interface{}{
"key2": 42,
"key3": true,
}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Empty input", func(t *testing.T) {
input := []interface{}{}
expected := map[string]interface{}{}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Nil input", func(t *testing.T) {
var input []interface{}
expected := map[string]interface{}{}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})
}
84 changes: 84 additions & 0 deletions logger/zerologadapter/adapter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

package zerologadapter_test

import (
"bytes"
"testing"

subject "go.mondoo.com/cnquery/v11/logger/zerologadapter"

"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
)

func TestNewAdapter(t *testing.T) {
var logOutput bytes.Buffer
logger := zerolog.New(&logOutput).Level(zerolog.DebugLevel)
adapter := subject.New(logger)

t.Run("Msg method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Msg("Test message", "key1", "value1", "key2", 42)

expectedLog := `{"level":"debug","key1":"value1","key2":42,"message":"Test message"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Error method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Error("Error occurred", "error_code", 500)

expectedLog := `{"level":"debug","error_code":500,"message":"Error occurred"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Info method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Info("Info message", "key", "value")

expectedLog := `{"level":"debug","key":"value","message":"Info message"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Debug method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Debug("Debugging issue", "context", "test")

expectedLog := `{"level":"debug","context":"test","message":"Debugging issue"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Warn method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Warn("Warning issued", "warning_level", "high")

expectedLog := `{"level":"debug","warning_level":"high","message":"Warning issued"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Handles non-string keys gracefully", func(t *testing.T) {
logOutput.Reset()
adapter.Info("Non-string key test", 123, "value", "key2", 42)

expectedLog := `{"level":"debug","key2":42,"message":"Non-string key test"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Handles odd number of key-value pairs gracefully", func(t *testing.T) {
logOutput.Reset()
adapter.Debug("Odd number test", "key1", "value1", "key2")

expectedLog := `{"level":"debug","key1":"value1","message":"Odd number test"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Empty key-value pairs", func(t *testing.T) {
logOutput.Reset()
adapter.Warn("Empty key-value test")

expectedLog := `{"level":"debug","message":"Empty key-value test"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})
}
33 changes: 2 additions & 31 deletions providers/aws/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ import (
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/hashicorp/go-retryablehttp"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/spf13/afero"
"go.mondoo.com/cnquery/v11/logger/zerologadapter"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/vault"
Expand Down Expand Up @@ -102,7 +102,7 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
// custom retry client
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 5
retryClient.Logger = &zeroLogAdapter{}
retryClient.Logger = zerologadapter.New(log.Logger)
c.awsConfigOptions = append(c.awsConfigOptions, config.WithHTTPClient(retryClient.StandardClient()))

cfg, err := config.LoadDefaultConfig(context.Background(), c.awsConfigOptions...)
Expand Down Expand Up @@ -398,32 +398,3 @@ func (h *AwsConnection) Regions() ([]string, error) {
h.clientcache.Store("_regions", &CacheEntry{Data: regions})
return regions, nil
}

// zeroLogAdapter is the adapter for retryablehttp is outputting debug messages
type zeroLogAdapter struct{}

func (l *zeroLogAdapter) Msg(msg string, keysAndValues ...interface{}) {
var e *zerolog.Event
// retry messages should only go to debug
e = log.Debug()
for i := 0; i < len(keysAndValues); i += 2 {
e = e.Interface(keysAndValues[i].(string), keysAndValues[i+1])
}
e.Msg(msg)
}

func (l *zeroLogAdapter) Error(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Info(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Debug(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Warn(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}
32 changes: 2 additions & 30 deletions providers/github/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/hashicorp/go-retryablehttp"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"go.mondoo.com/cnquery/v11/logger/zerologadapter"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/vault"
Expand Down Expand Up @@ -181,7 +182,7 @@ func newGithubTokenClient(conf *inventory.Config) (*github.Client, error) {
func newGithubRetryableClient(httpClient *http.Client) *http.Client {
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 5
retryClient.Logger = &zeroLogAdapter{}
retryClient.Logger = zerologadapter.New(log.Logger)

if httpClient == nil {
httpClient = http.DefaultClient
Expand Down Expand Up @@ -240,32 +241,3 @@ func newGithubRetryableClient(httpClient *http.Client) *http.Client {

return retryClient.StandardClient()
}

// zeroLogAdapter is the adapter for retryablehttp is outputting debug messages
type zeroLogAdapter struct{}

func (l *zeroLogAdapter) Msg(msg string, keysAndValues ...interface{}) {
var e *zerolog.Event
// retry messages should only go to debug
e = log.Debug()
for i := 0; i < len(keysAndValues); i += 2 {
e = e.Interface(keysAndValues[i].(string), keysAndValues[i+1])
}
e.Msg(msg)
}

func (l *zeroLogAdapter) Error(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Info(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Debug(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Warn(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}
36 changes: 2 additions & 34 deletions providers/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ import (

"github.com/cockroachdb/errors"
"github.com/hashicorp/go-retryablehttp"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/spf13/afero"
"github.com/ulikunitz/xz"
"go.mondoo.com/cnquery/v11/cli/config"
"go.mondoo.com/cnquery/v11/logger/zerologadapter"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/resources"
Expand Down Expand Up @@ -191,7 +191,7 @@ func httpClientWithRetry() (*http.Client, error) {

retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 3
retryClient.Logger = &ZerologAdapter{logger: log.Logger}
retryClient.Logger = zerologadapter.New(log.Logger)
retryClient.HTTPClient = &http.Client{
Transport: &http.Transport{
Proxy: proxyFn,
Expand Down Expand Up @@ -931,38 +931,6 @@ func MustLoadSchemaFromFile(name string, path string) *resources.Schema {
return MustLoadSchema(name, raw)
}

// ZerologAdapter adapts the zerolog logger to the LeveledLogger interface.
// Converts all retry logs to debug logs
type ZerologAdapter struct {
logger zerolog.Logger
}

func (z *ZerologAdapter) Error(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *ZerologAdapter) Info(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *ZerologAdapter) Debug(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *ZerologAdapter) Warn(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func convertToFields(keysAndValues ...interface{}) map[string]interface{} {
fields := make(map[string]interface{})
for i := 0; i < len(keysAndValues); i += 2 {
if i+1 < len(keysAndValues) {
fields[keysAndValues[i].(string)] = keysAndValues[i+1]
}
}
return fields
}

func LoadAssetUrlSchema() (*inventory.AssetUrlSchema, error) {
providers, err := ListAll()
if err != nil {
Expand Down
Loading

0 comments on commit 55dbd36

Please sign in to comment.