Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🛠️ new logger/zerologadapter for retryable requests #4903

Merged
merged 3 commits into from
Nov 21, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions providers-sdk/v1/util/zerologadapter/adapter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

package zerologadapter
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I put it inside the providers-sdk/v1 to make it clear that providers can and should use it, Ill move it though.


import "github.com/rs/zerolog"

// New returns a new adapter for the zerolog logger to the LeveledLogger interface. This struct is
// mainly used in conjunction with a retryable http client to convert all retry logs to debug logs.
//
// NOTE that all messages will go to debug level.
//
// e.g.
// ```go
// retryClient := retryablehttp.NewClient()
// retryClient.Logger = zerologadapter.New(log.Logger)
// ```
func New(logger zerolog.Logger) *Adapter {
return &Adapter{logger}
}

type Adapter struct {
logger zerolog.Logger
}

func (z *Adapter) Msg(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Error(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Info(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Debug(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *Adapter) Warn(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func convertToFields(keysAndValues ...interface{}) map[string]interface{} {
fields := make(map[string]interface{})
for i := 0; i < len(keysAndValues); i += 2 {
if i+1 < len(keysAndValues) {
keyString, ok := keysAndValues[i].(string)
if ok { // safelty first, eventhough we always expect a string
afiune marked this conversation as resolved.
Show resolved Hide resolved
fields[keyString] = keysAndValues[i+1]
}
}
}
return fields
}
61 changes: 61 additions & 0 deletions providers-sdk/v1/util/zerologadapter/adapter_internal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

package zerologadapter

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestConvertToFields(t *testing.T) {
t.Run("Valid key-value pairs", func(t *testing.T) {
input := []interface{}{"key1", "value1", "key2", 42, "key3", true}
expected := map[string]interface{}{
"key1": "value1",
"key2": 42,
"key3": true,
}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Odd number of elements", func(t *testing.T) {
input := []interface{}{"key1", "value1", "key2"}
expected := map[string]interface{}{
"key1": "value1",
}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Non-string keys are ignored", func(t *testing.T) {
input := []interface{}{123, "value1", "key2", 42, 3.14, "value3", "key3", true}
expected := map[string]interface{}{
"key2": 42,
"key3": true,
}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Empty input", func(t *testing.T) {
input := []interface{}{}
expected := map[string]interface{}{}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})

t.Run("Nil input", func(t *testing.T) {
var input []interface{}
expected := map[string]interface{}{}

result := convertToFields(input...)
assert.Equal(t, expected, result)
})
}
84 changes: 84 additions & 0 deletions providers-sdk/v1/util/zerologadapter/adapter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

package zerologadapter_test

import (
"bytes"
"testing"

subject "go.mondoo.com/cnquery/v11/providers-sdk/v1/util/zerologadapter"

"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
)

func TestNewAdapter(t *testing.T) {
var logOutput bytes.Buffer
logger := zerolog.New(&logOutput).Level(zerolog.DebugLevel)
adapter := subject.New(logger)

t.Run("Msg method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Msg("Test message", "key1", "value1", "key2", 42)

expectedLog := `{"level":"debug","key1":"value1","key2":42,"message":"Test message"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Error method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Error("Error occurred", "error_code", 500)

expectedLog := `{"level":"debug","error_code":500,"message":"Error occurred"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Info method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Info("Info message", "key", "value")

expectedLog := `{"level":"debug","key":"value","message":"Info message"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Debug method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Debug("Debugging issue", "context", "test")

expectedLog := `{"level":"debug","context":"test","message":"Debugging issue"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Warn method logs correctly", func(t *testing.T) {
logOutput.Reset()
adapter.Warn("Warning issued", "warning_level", "high")

expectedLog := `{"level":"debug","warning_level":"high","message":"Warning issued"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Handles non-string keys gracefully", func(t *testing.T) {
logOutput.Reset()
adapter.Info("Non-string key test", 123, "value", "key2", 42)

expectedLog := `{"level":"debug","key2":42,"message":"Non-string key test"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Handles odd number of key-value pairs gracefully", func(t *testing.T) {
logOutput.Reset()
adapter.Debug("Odd number test", "key1", "value1", "key2")

expectedLog := `{"level":"debug","key1":"value1","message":"Odd number test"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})

t.Run("Empty key-value pairs", func(t *testing.T) {
logOutput.Reset()
adapter.Warn("Empty key-value test")

expectedLog := `{"level":"debug","message":"Empty key-value test"}`
assert.JSONEq(t, expectedLog, logOutput.String())
})
}
33 changes: 2 additions & 31 deletions providers/aws/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ import (
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/hashicorp/go-retryablehttp"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/spf13/afero"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/util/zerologadapter"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/vault"
"go.mondoo.com/cnquery/v11/providers/os/connection/shared"
)
Expand Down Expand Up @@ -102,7 +102,7 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
// custom retry client
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 5
retryClient.Logger = &zeroLogAdapter{}
retryClient.Logger = zerologadapter.New(log.Logger)
c.awsConfigOptions = append(c.awsConfigOptions, config.WithHTTPClient(retryClient.StandardClient()))

cfg, err := config.LoadDefaultConfig(context.Background(), c.awsConfigOptions...)
Expand Down Expand Up @@ -398,32 +398,3 @@ func (h *AwsConnection) Regions() ([]string, error) {
h.clientcache.Store("_regions", &CacheEntry{Data: regions})
return regions, nil
}

// zeroLogAdapter is the adapter for retryablehttp is outputting debug messages
type zeroLogAdapter struct{}

func (l *zeroLogAdapter) Msg(msg string, keysAndValues ...interface{}) {
var e *zerolog.Event
// retry messages should only go to debug
e = log.Debug()
for i := 0; i < len(keysAndValues); i += 2 {
e = e.Interface(keysAndValues[i].(string), keysAndValues[i+1])
}
e.Msg(msg)
}

func (l *zeroLogAdapter) Error(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Info(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Debug(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Warn(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}
32 changes: 2 additions & 30 deletions providers/github/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/rs/zerolog/log"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/util/zerologadapter"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/vault"
"golang.org/x/oauth2"
)
Expand Down Expand Up @@ -181,7 +182,7 @@ func newGithubTokenClient(conf *inventory.Config) (*github.Client, error) {
func newGithubRetryableClient(httpClient *http.Client) *http.Client {
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 5
retryClient.Logger = &zeroLogAdapter{}
retryClient.Logger = zerologadapter.New(log.Logger)

if httpClient == nil {
httpClient = http.DefaultClient
Expand Down Expand Up @@ -240,32 +241,3 @@ func newGithubRetryableClient(httpClient *http.Client) *http.Client {

return retryClient.StandardClient()
}

// zeroLogAdapter is the adapter for retryablehttp is outputting debug messages
type zeroLogAdapter struct{}

func (l *zeroLogAdapter) Msg(msg string, keysAndValues ...interface{}) {
var e *zerolog.Event
// retry messages should only go to debug
e = log.Debug()
for i := 0; i < len(keysAndValues); i += 2 {
e = e.Interface(keysAndValues[i].(string), keysAndValues[i+1])
}
e.Msg(msg)
}

func (l *zeroLogAdapter) Error(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Info(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Debug(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}

func (l *zeroLogAdapter) Warn(msg string, keysAndValues ...interface{}) {
l.Msg(msg, keysAndValues...)
}
36 changes: 2 additions & 34 deletions providers/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ import (

"github.com/cockroachdb/errors"
"github.com/hashicorp/go-retryablehttp"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/spf13/afero"
"github.com/ulikunitz/xz"
"go.mondoo.com/cnquery/v11/cli/config"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/resources"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/util/zerologadapter"
"go.mondoo.com/cnquery/v11/providers/core/resources/versions/semver"
"golang.org/x/exp/slices"
)
Expand Down Expand Up @@ -191,7 +191,7 @@ func httpClientWithRetry() (*http.Client, error) {

retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 3
retryClient.Logger = &ZerologAdapter{logger: log.Logger}
retryClient.Logger = zerologadapter.New(log.Logger)
retryClient.HTTPClient = &http.Client{
Transport: &http.Transport{
Proxy: proxyFn,
Expand Down Expand Up @@ -931,38 +931,6 @@ func MustLoadSchemaFromFile(name string, path string) *resources.Schema {
return MustLoadSchema(name, raw)
}

// ZerologAdapter adapts the zerolog logger to the LeveledLogger interface.
// Converts all retry logs to debug logs
type ZerologAdapter struct {
logger zerolog.Logger
}

func (z *ZerologAdapter) Error(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *ZerologAdapter) Info(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *ZerologAdapter) Debug(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func (z *ZerologAdapter) Warn(msg string, keysAndValues ...interface{}) {
z.logger.Debug().Fields(convertToFields(keysAndValues...)).Msg(msg)
}

func convertToFields(keysAndValues ...interface{}) map[string]interface{} {
fields := make(map[string]interface{})
for i := 0; i < len(keysAndValues); i += 2 {
if i+1 < len(keysAndValues) {
fields[keysAndValues[i].(string)] = keysAndValues[i+1]
}
}
return fields
}

func LoadAssetUrlSchema() (*inventory.AssetUrlSchema, error) {
providers, err := ListAll()
if err != nil {
Expand Down
Loading
Loading