diff --git a/encoder/encoder.go b/encoder/encoder.go new file mode 100644 index 000000000..dd9266f1a --- /dev/null +++ b/encoder/encoder.go @@ -0,0 +1,214 @@ +// Copyright (c) 2023 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package encoder + +import ( + "strconv" + "strings" + "time" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// Tags consists of list of Tag +type Tags []Tag + +// Tag is a key value pair +type Tag struct { + Key string `json:"key,required"` + Value string `json:"value,required"` +} + +// StringTagEncoder is an ObjectEncoder backed by a slice []*Tag +type StringTagEncoder struct { + fields map[string]*Tag + prefixes []string +} + +// NewStringTagEncoder creates a new slice backed ObjectEncoder. +func NewStringTagEncoder(prefixes ...string) *StringTagEncoder { + return &StringTagEncoder{ + fields: map[string]*Tag{}, + prefixes: prefixes, + } +} + +// GetTags returns a slice of tags +func (s *StringTagEncoder) GetTags() []*Tag { + tags := make([]*Tag, 0, len(s.fields)) + for _, t := range s.fields { + tag := t + tags = append(tags, tag) + } + return tags +} + +// AddArray implements ObjectEncoder. +func (s *StringTagEncoder) AddArray(key string, v zapcore.ArrayMarshaler) error { + arr := &sliceArrayEncoder{} + if err := v.MarshalLogArray(arr); err != nil { + return err + } + for _, elem := range arr.elems { + field := zap.Any(key, elem) + field.AddTo(s) + } + return nil +} + +// AddObject implements ObjectEncoder. +func (s *StringTagEncoder) AddObject(k string, v zapcore.ObjectMarshaler) error { + newMap := NewStringTagEncoder(k) + if err := v.MarshalLogObject(newMap); err != nil { + return err + } + for key, tag := range newMap.fields { + s.fields[key] = tag + } + return nil +} + +// AddBinary implements ObjectEncoder. +func (s *StringTagEncoder) AddBinary(k string, v []byte) { return } //encoder will ignore non string binaries + +// AddByteString implements ObjectEncoder. +func (s *StringTagEncoder) AddByteString(k string, v []byte) { s.AddString(k, string(v)) } + +// AddBool implements ObjectEncoder. +func (s *StringTagEncoder) AddBool(k string, v bool) { s.AddString(k, strconv.FormatBool(v)) } + +// AddDuration implements ObjectEncoder. +func (s *StringTagEncoder) AddDuration(k string, v time.Duration) { s.AddString(k, v.String()) } + +// AddComplex128 implements ObjectEncoder. +func (s *StringTagEncoder) AddComplex128(k string, v complex128) { return } //encoder will ignore complex128 + +// AddComplex64 implements ObjectEncoder. +func (s *StringTagEncoder) AddComplex64(k string, v complex64) { return } // encoder will ignore complex64 + +// AddFloat64 implements ObjectEncoder. +func (s *StringTagEncoder) AddFloat64(k string, v float64) { + s.AddString(k, strconv.FormatFloat(v, 'f', -1, 64)) +} + +// AddFloat32 implements ObjectEncoder. +func (s *StringTagEncoder) AddFloat32(k string, v float32) { s.AddFloat64(k, float64(v)) } + +// AddInt implements ObjectEncoder. +func (s *StringTagEncoder) AddInt(k string, v int) { s.AddString(k, strconv.Itoa(v)) } + +// AddInt64 implements ObjectEncoder. +func (s *StringTagEncoder) AddInt64(k string, v int64) { s.AddInt(k, int(v)) } + +// AddInt32 implements ObjectEncoder. +func (s *StringTagEncoder) AddInt32(k string, v int32) { s.AddInt(k, int(v)) } + +// AddInt16 implements ObjectEncoder. +func (s *StringTagEncoder) AddInt16(k string, v int16) { s.AddInt(k, int(v)) } + +// AddInt8 implements ObjectEncoder. +func (s *StringTagEncoder) AddInt8(k string, v int8) { s.AddInt(k, int(v)) } + +// AddString implements ObjectEncoder. +func (s *StringTagEncoder) AddString(k string, v string) { + s.fields[k] = &Tag{Key: s.key(k), Value: v} +} + +// AddTime implements ObjectEncoder. +func (s *StringTagEncoder) AddTime(k string, v time.Time) { s.AddString(k, v.String()) } + +// AddUint implements ObjectEncoder. +func (s *StringTagEncoder) AddUint(k string, v uint) { s.AddUint64(k, uint64(v)) } + +// AddUint64 implements ObjectEncoder. +func (s *StringTagEncoder) AddUint64(k string, v uint64) { s.AddString(k, strconv.FormatUint(v, 10)) } + +// AddUint32 implements ObjectEncoder. +func (s *StringTagEncoder) AddUint32(k string, v uint32) { s.AddUint64(k, uint64(v)) } + +// AddUint16 implements ObjectEncoder. +func (s *StringTagEncoder) AddUint16(k string, v uint16) { s.AddUint64(k, uint64(v)) } + +// AddUint8 implements ObjectEncoder. +func (s *StringTagEncoder) AddUint8(k string, v uint8) { s.AddUint64(k, uint64(v)) } + +// AddUintptr implements ObjectEncoder. +func (s *StringTagEncoder) AddUintptr(k string, v uintptr) { return } //encoder will ignore uintptr + +// AddReflected implements ObjectEncoder. +func (s *StringTagEncoder) AddReflected(k string, v interface{}) error { return nil } //encoder will ignore reflected values + +// OpenNamespace implements ObjectEncoder. +func (s *StringTagEncoder) OpenNamespace(k string) { + s.prefixes = append(s.prefixes, k) +} + +func (s *StringTagEncoder) key(k string) string { + return strings.Join(append(s.prefixes, k), ".") +} + +// sliceArrayEncoder is an ArrayEncoder backed by a simple []interface{}. Like +// the MapObjectEncoder, it's not designed for production use. +type sliceArrayEncoder struct { + elems []interface{} +} + +func (s *sliceArrayEncoder) AppendArray(v zapcore.ArrayMarshaler) error { + enc := &sliceArrayEncoder{} + err := v.MarshalLogArray(enc) + s.elems = append(s.elems, enc.elems) + return err +} + +func (s *sliceArrayEncoder) AppendObject(v zapcore.ObjectMarshaler) error { + enc := NewStringTagEncoder() + err := v.MarshalLogObject(enc) + for _, f := range enc.fields { + s.elems = append(s.elems, f.Value) + } + return err +} + +func (s *sliceArrayEncoder) AppendReflected(v interface{}) error { + s.elems = append(s.elems, v) + return nil +} + +func (s *sliceArrayEncoder) AppendBool(v bool) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendByteString(v []byte) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendComplex128(v complex128) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendComplex64(v complex64) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendDuration(v time.Duration) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendFloat64(v float64) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendFloat32(v float32) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendInt(v int) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendInt64(v int64) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendInt32(v int32) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendInt16(v int16) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendInt8(v int8) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendString(v string) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendTime(v time.Time) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendUint(v uint) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendUint64(v uint64) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendUint32(v uint32) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendUint16(v uint16) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendUint8(v uint8) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendUintptr(v uintptr) { s.elems = append(s.elems, v) } diff --git a/encoder/encoder_test.go b/encoder/encoder_test.go new file mode 100644 index 000000000..574313515 --- /dev/null +++ b/encoder/encoder_test.go @@ -0,0 +1,330 @@ +// Copyright (c) 2023 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package encoder + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + "unsafe" + + "github.com/opentracing/opentracing-go" + "github.com/stretchr/testify/assert" + "github.com/uber/jaeger-client-go" + jzap "github.com/uber/jaeger-client-go/log/zap" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func TestStringTagEncoder(t *testing.T) { + timeString := time.Unix(0, 0) + span := &jaeger.Span{} + ctx := jaeger.NewSpanContext( + jaeger.TraceID{High: 5678, Low: 1234}, + jaeger.SpanID(9), + jaeger.SpanID(10), + false, + map[string]string{}, + ) + ptr := reflect.ValueOf(span) + val := reflect.Indirect(ptr) + //set operationName + f := val.FieldByName("context") + f = reflect.NewAt(f.Type(), unsafe.Pointer(f.UnsafeAddr())).Elem() + v := reflect.ValueOf(ctx) + f.Set(v) + ctxWitSpan := opentracing.ContextWithSpan(context.Background(), span) + tests := []struct { + name string + prefixes []string + fields []zapcore.Field + expected map[string][]string + }{ + + { + name: "should encode string", + fields: []zapcore.Field{ + zap.Any("foo", "bar"), + }, + expected: map[string][]string{ + "foo": {"bar"}, + }, + }, + { + name: "should encode array", + fields: []zapcore.Field{ + zap.Strings("foo", []string{"bar", "boo"}), + }, + expected: map[string][]string{ + "foo": {"bar", "boo"}, + }, + }, + { + name: "should encode object", + fields: []zapcore.Field{ + jzap.Trace(ctxWitSpan), + }, + expected: map[string][]string{ + "trace.trace": {jaeger.TraceID{High: 5678, Low: 1234}.String()}, + "trace.span": {jaeger.SpanID(9).String()}, + }, + }, + { + name: "should encode with open namespace", + fields: []zapcore.Field{ + zap.Any("foo", int64(1234)), + { + Key: "boo", + Type: zapcore.NamespaceType, + }, + zap.Any("foo", int64(1234)), + }, + expected: map[string][]string{ + "foo": {"1234"}, + "boo.foo": {"1234"}, + }, + }, + { + name: "should skip binary", + fields: []zapcore.Field{ + zap.Any("foo", []uint8{}), + }, + expected: map[string][]string{}, + }, + { + name: "should encode binary string", + fields: []zapcore.Field{ + { + Key: "foo", + Type: zapcore.ByteStringType, + Interface: []byte("bar"), + }, + }, + expected: map[string][]string{ + "foo": {"bar"}, + }, + }, + { + name: "should encode bool", + fields: []zapcore.Field{ + zap.Any("foo", true), + }, + expected: map[string][]string{ + "foo": {"true"}, + }, + }, + { + name: "should encode duration", + fields: []zapcore.Field{ + zap.Any("foo", time.Nanosecond), + }, + expected: map[string][]string{ + "foo": {"1ns"}, + }, + }, + { + name: "should append errors", + fields: []zapcore.Field{ + zap.Errors("errors", []error{errors.New("bar"), errors.New("boo")}), + }, + expected: map[string][]string{ + "errors": {"bar", "boo"}, + }, + }, + { + name: "should encode complex128", + fields: []zapcore.Field{ + zap.Any("foo", complex128(12)), + }, + expected: map[string][]string{}, + }, + { + name: "should encode complex64", + fields: []zapcore.Field{ + zap.Any("foo", complex64(12)), + }, + expected: map[string][]string{}, + }, + { + name: "should encode float64", + fields: []zapcore.Field{ + zap.Any("foo", float64(12)), + }, + expected: map[string][]string{ + "foo": {"12"}, + }, + }, + { + name: "should encode float64", + fields: []zapcore.Field{ + zap.Any("foo", float32(12)), + }, + expected: map[string][]string{ + "foo": {"12"}, + }, + }, + { + name: "should encode int", + fields: []zapcore.Field{ + zap.Any("foo", int(1234)), + }, + expected: map[string][]string{ + "foo": {"1234"}, + }, + }, + { + name: "should encode int64", + fields: []zapcore.Field{ + zap.Any("foo", int64(1234)), + }, + expected: map[string][]string{ + "foo": {"1234"}, + }, + }, + { + name: "should encode int32", + fields: []zapcore.Field{ + zap.Any("foo", int32(1234)), + }, + expected: map[string][]string{ + "foo": {"1234"}, + }, + }, + { + name: "should encode int16", + fields: []zapcore.Field{ + zap.Any("foo", int16(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode int8", + fields: []zapcore.Field{ + zap.Any("foo", int8(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode uint", + fields: []zapcore.Field{ + zap.Any("foo", uint(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode uint8", + fields: []zapcore.Field{ + zap.Any("foo", uint8(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode uint16", + fields: []zapcore.Field{ + zap.Any("foo", uint16(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode uint32", + fields: []zapcore.Field{ + zap.Any("foo", uint32(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode uint64", + fields: []zapcore.Field{ + zap.Any("foo", uint64(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode uintptr", + fields: []zapcore.Field{ + zap.Any("foo", uintptr(1)), + }, + expected: map[string][]string{ + "foo": {"1"}, + }, + }, + { + name: "should encode time", + fields: []zapcore.Field{ + zap.Any("foo", timeString), + }, + expected: map[string][]string{ + "foo": {timeString.String()}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + enc := NewStringTagEncoder(tt.prefixes...) + for _, field := range tt.fields { + field.AddTo(enc) + } + for _, field := range enc.fields { + assert.Contains(t, tt.expected[field.Key], field.Value) + } + }) + } +} + +func TestSliceArrayEncoder_Appends(t *testing.T) { + enc := sliceArrayEncoder{} + enc.AppendBool(true) + enc.AppendByteString([]byte{}) + enc.AppendComplex64(complex64(1)) + enc.AppendComplex128(complex128(1)) + enc.AppendDuration(time.Second) + enc.AppendFloat32(float32(1)) + enc.AppendFloat64(float64(1)) + enc.AppendInt(int(1)) + enc.AppendInt8(int8(1)) + enc.AppendInt16(int16(1)) + enc.AppendInt32(int32(1)) + enc.AppendInt64(int64(1)) + enc.AppendString("") + enc.AppendTime(time.Now()) + enc.AppendUint(uint(1)) + enc.AppendUint8(uint8(1)) + enc.AppendUint16(uint16(1)) + enc.AppendUint32(uint32(1)) + enc.AppendUint64(uint64(1)) + enc.AppendUintptr(uintptr(1)) +} diff --git a/runtime/context.go b/runtime/context.go index b6b0bf079..d3b54de3c 100644 --- a/runtime/context.go +++ b/runtime/context.go @@ -22,10 +22,12 @@ package zanzibar import ( "context" + "encoding/json" "strconv" "time" "github.com/uber-go/tally" + "github.com/uber/zanzibar/encoder" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -231,7 +233,15 @@ func accumulateLogMsgAndFieldsInContext(ctx context.Context, msg string, newFiel logLevel = logLevelValue } } - ctx = WithLogFields(ctx, zap.String("msg"+strconv.Itoa(ctxLogCounter), msg)) + zapFields := make([]zap.Field, len(newFields)) + copy(zapFields, newFields) + zapFields = append(zapFields, zap.String("msg", msg)) + msgBytes, err := json.Marshal(GetTagsFromZapFields(zapFields...)) + if err != nil { + ctx = WithLogFields(ctx, zap.String("msg"+strconv.Itoa(ctxLogCounter), msg)) + } else { + ctx = WithLogFields(ctx, zap.String("msg"+strconv.Itoa(ctxLogCounter), string(msgBytes))) + } ctx = WithLogFields(ctx, newFields...) ctx = context.WithValue(ctx, ctxLogCounterName, ctxLogCounter) ctx = context.WithValue(ctx, ctxLogLevel, logLevel) @@ -293,6 +303,26 @@ func (c *ContextExtractors) ExtractLogFields(ctx context.Context) []zap.Field { return fields } +// GetTagsFromZapFields Get all tags from zap fields +func GetTagsFromZapFields(fs ...zapcore.Field) encoder.Tags { + enc := encoder.NewStringTagEncoder() + tags := encoder.Tags(make([]encoder.Tag, 0, len(fs))) + for _, f := range fs { + f.AddTo(enc) + } + for _, t := range enc.GetTags() { + if t == nil { + continue + } + tag := encoder.Tag{ + Key: t.Key, + Value: t.Value, + } + tags = append(tags, tag) + } + return tags +} + // ContextLogger is a logger that extracts some log fields from the context before passing through to underlying zap logger. // In cases it also updates the context instead of logging type ContextLogger interface { @@ -326,7 +356,7 @@ func NewContextLogger(log *zap.Logger) ContextLogger { } } -//GetLogger returns the logger +// GetLogger returns the logger func (c *contextLogger) GetLogger() Logger { return c.log } diff --git a/runtime/context_test.go b/runtime/context_test.go index 904fb458e..800444225 100644 --- a/runtime/context_test.go +++ b/runtime/context_test.go @@ -329,12 +329,11 @@ func TestAccumulateLogMsgAndFieldsInContext(t *testing.T) { ctx = accumulateLogMsgAndFieldsInContext(ctx, "message2", []zap.Field{zap.String("ctxField1", "ctxFieldValue2")}, zapcore.ErrorLevel) logFields := GetLogFieldsFromCtx(ctx) - assert.Equal(t, []zap.Field{ - zap.String("msg1", "message1"), - zap.String("ctxField1", "ctxFieldValue1"), - zap.String("msg2", "message2"), - zap.String("ctxField1", "ctxFieldValue2"), - }, logFields) + var keys []string + for _, f := range logFields { + keys = append(keys, f.Key) + } + assert.Equal(t, []string{"msg1", "ctxField1", "msg2", "ctxField1"}, keys) } func TestAccumulateLogMsgAndFieldsInContextWithLogLevel(t *testing.T) {