From bbaab93036942f262d36be369ad39dc3cdfa0011 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 03:57:05 -0600 Subject: [PATCH 01/11] Update readme with new project references --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c7904ca..7641f9f 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ With this in mind, this repo was created with several different ways of deduplic - [slogotel](https://github.com/veqryn/slog-context/tree/main/otel): Automatically extract and add [OpenTelemetry](https://opentelemetry.io/) TraceID's to all log lines. - [slogdedup](https://github.com/veqryn/slog-dedup): Middleware that deduplicates and sorts attributes. Particularly useful for JSON logging. - [slogbugsnag](https://github.com/veqryn/slog-bugsnag): Middleware that pipes Errors to [Bugsnag](https://www.bugsnag.com/). +- [slogjson](https://github.com/veqryn/slog-json): Formatter that uses the [JSON v2](https://github.com/golang/go/discussions/63397) [library](https://github.com/go-json-experiment/json), with optional single-line pretty-printing. ## Install `go get github.com/veqryn/slog-dedup` From 8d40ca2a456cfc581eb005dcd331d204b3093ae3 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 03:58:08 -0600 Subject: [PATCH 02/11] Update increment and overwrite handlers with new resolve key method --- increment_handler.go | 82 +++++++++++++++++++++++--------------------- overwrite_handler.go | 31 +++++++++-------- 2 files changed, 59 insertions(+), 54 deletions(-) diff --git a/increment_handler.go b/increment_handler.go index f76d66e..de9cc53 100644 --- a/increment_handler.go +++ b/increment_handler.go @@ -3,6 +3,7 @@ package slogdedup import ( "context" "log/slog" + "slices" "modernc.org/b/v2" ) @@ -14,20 +15,25 @@ type IncrementHandlerOptions struct { // Function that will only be called on all root level (not in a group) attribute keys. // Returns true if the key conflicts with the builtin keys. - DoesBuiltinKeyConflict func(key string) bool + //DoesBuiltinKeyConflict func(key string) bool // IncrementKeyName should return a modified key string based on the index (first, second, third instance seen, etc) - IncrementKeyName func(key string, index int) string + //IncrementKeyName func(key string, index int) string + + // Function that will be called on all root level (not in a group) attribute keys. + // Returns the new key value to use, and true to keep the attribute or false to drop it. + // Can be used to drop, keep, or rename any attributes matching the builtin attributes. + ResolveKey func(groups []string, key string, index int) (string, bool) } // IncrementHandler is a slog.Handler middleware that will deduplicate all attributes and // groups by incrementing/modifying their key names. // It passes the final record and attributes off to the next handler when finished. type IncrementHandler struct { - next slog.Handler - goa *groupOrAttrs - keyCompare func(a, b string) int - getIncrementKey func(uniq *b.Tree[string, any], depth int, key string) string + next slog.Handler + goa *groupOrAttrs + keyCompare func(a, b string) int + resolveIncrementKey func(uniq *b.Tree[string, any], groups []string, key string) (string, bool) } var _ slog.Handler = &IncrementHandler{} // Assert conformance with interface @@ -61,17 +67,14 @@ func NewIncrementHandler(next slog.Handler, opts *IncrementHandlerOptions) *Incr if opts.KeyCompare == nil { opts.KeyCompare = CaseSensitiveCmp } - if opts.DoesBuiltinKeyConflict == nil { - opts.DoesBuiltinKeyConflict = DoesBuiltinKeyConflict - } - if opts.IncrementKeyName == nil { - opts.IncrementKeyName = IncrementKeyName + if opts.ResolveKey == nil { + opts.ResolveKey = IncrementIfBuiltinKeyConflict } return &IncrementHandler{ - next: next, - keyCompare: opts.KeyCompare, - getIncrementKey: seekIncrementKeyClosure(opts.DoesBuiltinKeyConflict, opts.IncrementKeyName), + next: next, + keyCompare: opts.KeyCompare, + resolveIncrementKey: resolveIncrementKeyClosure(opts.ResolveKey), } } @@ -94,7 +97,7 @@ func (h *IncrementHandler) Handle(ctx context.Context, r slog.Record) error { // Resolve groups and with-attributes uniq := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniq, goas, 0) + h.createAttrTree(uniq, goas, nil) // Add all attributes to new record (because old record has all the old attributes) newR := &slog.Record{ @@ -126,32 +129,34 @@ func (h *IncrementHandler) WithAttrs(attrs []slog.Attr) slog.Handler { // createAttrTree recursively goes through all groupOrAttrs, resolving their attributes and creating subtrees as // necessary, adding the results to the map -func (h *IncrementHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, depth int) { +func (h *IncrementHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, groups []string) { if len(goas) == 0 { return } // If a group is encountered, create a subtree for that group and all groupOrAttrs after it if goas[0].group != "" { - goas[0].group = h.getIncrementKey(uniq, depth, goas[0].group) - uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniqGroup, goas[1:], depth+1) - // Ignore empty groups, otherwise put subtree into the map - if uniqGroup.Len() > 0 { - uniq.Set(goas[0].group, uniqGroup) + if key, keep := h.resolveIncrementKey(uniq, groups, goas[0].group); keep { + uniqGroup := b.TreeNew[string, any](h.keyCompare) + h.createAttrTree(uniqGroup, goas[1:], append(slices.Clip(groups), key)) + // Ignore empty groups, otherwise put subtree into the map + if uniqGroup.Len() > 0 { + uniq.Set(key, uniqGroup) + } + return } - return } // Otherwise, set all attributes for this groupOrAttrs, and then call again for remaining groupOrAttrs's - h.resolveValues(uniq, goas[0].attrs, depth) - h.createAttrTree(uniq, goas[1:], depth) + h.resolveValues(uniq, goas[0].attrs, groups) + h.createAttrTree(uniq, goas[1:], groups) } // resolveValues iterates through the attributes, resolving them and putting them into the map. // If a group is encountered (as an attribute), it will be separately resolved and added as a subtree. // Since attributes are ordered from oldest to newest, it increments the key names as it goes. -func (h *IncrementHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, depth int) { +func (h *IncrementHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, groups []string) { + var ok bool for _, a := range attrs { a.Value = a.Value.Resolve() if a.Equal(slog.Attr{}) { @@ -159,7 +164,10 @@ func (h *IncrementHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog } // Default situation: resolve the key and put it into the map - a.Key = h.getIncrementKey(uniq, depth, a.Key) + a.Key, ok = h.resolveIncrementKey(uniq, groups, a.Key) + if !ok { + continue + } if a.Value.Kind() != slog.KindGroup { uniq.Set(a.Key, a) @@ -168,13 +176,13 @@ func (h *IncrementHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog // Groups with empty keys are inlined if a.Key == "" { - h.resolveValues(uniq, a.Value.Group(), depth) + h.resolveValues(uniq, a.Value.Group(), groups) continue } // Create a subtree for this group uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.resolveValues(uniqGroup, a.Value.Group(), depth+1) + h.resolveValues(uniqGroup, a.Value.Group(), append(slices.Clip(groups), a.Key)) // Ignore empty groups, otherwise put subtree into the map if uniqGroup.Len() > 0 { @@ -183,15 +191,11 @@ func (h *IncrementHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog } } -// seekIncrementKeyClosure returns a function to be used to resolve a key for IncrementHandler. -func seekIncrementKeyClosure(doesBuiltinKeyConflict func(key string) bool, incrementKeyName func(key string, index int) string) func(uniq *b.Tree[string, any], depth int, key string) string { - return func(uniq *b.Tree[string, any], depth int, key string) string { +// resolveIncrementKeyClosure returns a function to be used to resolve a key for IncrementHandler. +func resolveIncrementKeyClosure(resolveKey func(groups []string, key string, index int) (string, bool)) func(uniq *b.Tree[string, any], groups []string, key string) (string, bool) { + return func(uniq *b.Tree[string, any], groups []string, key string) (string, bool) { var index int - if depth == 0 && doesBuiltinKeyConflict(key) { - index++ // Don't overwrite the built-in attribute keys - } - - newKey := incrementKeyName(key, index) + newKey, keep := resolveKey(groups, key, index) // Seek cursor to the key in the map equal to or less than newKey en, _ := uniq.Seek(newKey) @@ -201,12 +205,12 @@ func seekIncrementKeyClosure(doesBuiltinKeyConflict func(key string) bool, incre for { k, _, err := en.Next() if err != nil || k > newKey { - return newKey + return newKey, keep } if k == newKey { // If the next key is equal to newKey, we must increment our key index++ - newKey = incrementKeyName(key, index) + newKey, keep = resolveKey(groups, key, index) } } } diff --git a/overwrite_handler.go b/overwrite_handler.go index a60d83f..e646d5f 100644 --- a/overwrite_handler.go +++ b/overwrite_handler.go @@ -3,6 +3,7 @@ package slogdedup import ( "context" "log/slog" + "slices" "modernc.org/b/v2" ) @@ -15,7 +16,7 @@ type OverwriteHandlerOptions struct { // Function that will be called on all root level (not in a group) attribute keys. // Returns the new key value to use, and true to keep the attribute or false to drop it. // Can be used to drop, keep, or rename any attributes matching the builtin attributes. - ResolveBuiltinKeyConflict func(k string) (string, bool) + ResolveKey func(groups []string, key string, _ int) (string, bool) } // OverwriteHandler is a slog.Handler middleware that will deduplicate all attributes and @@ -25,7 +26,7 @@ type OverwriteHandler struct { next slog.Handler goa *groupOrAttrs keyCompare func(a, b string) int - getKey func(key string, depth int) (string, bool) + resolveKey func(groups []string, key string, _ int) (string, bool) } var _ slog.Handler = &OverwriteHandler{} // Assert conformance with interface @@ -59,14 +60,14 @@ func NewOverwriteHandler(next slog.Handler, opts *OverwriteHandlerOptions) *Over if opts.KeyCompare == nil { opts.KeyCompare = CaseSensitiveCmp } - if opts.ResolveBuiltinKeyConflict == nil { - opts.ResolveBuiltinKeyConflict = IncrementIfBuiltinKeyConflict + if opts.ResolveKey == nil { + opts.ResolveKey = IncrementIfBuiltinKeyConflict } return &OverwriteHandler{ next: next, keyCompare: opts.KeyCompare, - getKey: getKeyClosure(opts.ResolveBuiltinKeyConflict), + resolveKey: opts.ResolveKey, } } @@ -89,7 +90,7 @@ func (h *OverwriteHandler) Handle(ctx context.Context, r slog.Record) error { // Resolve groups and with-attributes uniq := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniq, goas, 0) + h.createAttrTree(uniq, goas, nil) // Add all attributes to new record (because old record has all the old attributes) newR := &slog.Record{ @@ -121,16 +122,16 @@ func (h *OverwriteHandler) WithAttrs(attrs []slog.Attr) slog.Handler { // createAttrTree recursively goes through all groupOrAttrs, resolving their attributes and creating subtrees as // necessary, adding the results to the map -func (h *OverwriteHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, depth int) { +func (h *OverwriteHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, groups []string) { if len(goas) == 0 { return } // If a group is encountered, create a subtree for that group and all groupOrAttrs after it if goas[0].group != "" { - if key, ok := h.getKey(goas[0].group, depth); ok { + if key, ok := h.resolveKey(groups, goas[0].group, 0); ok { uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniqGroup, goas[1:], depth+1) + h.createAttrTree(uniqGroup, goas[1:], append(slices.Clip(groups), key)) // Ignore empty groups, otherwise put subtree into the map if uniqGroup.Len() > 0 { uniq.Set(key, uniqGroup) @@ -140,14 +141,14 @@ func (h *OverwriteHandler) createAttrTree(uniq *b.Tree[string, any], goas []*gro } // Otherwise, set all attributes for this groupOrAttrs, and then call again for remaining groupOrAttrs's - h.resolveValues(uniq, goas[0].attrs, depth) - h.createAttrTree(uniq, goas[1:], depth) + h.resolveValues(uniq, goas[0].attrs, groups) + h.createAttrTree(uniq, goas[1:], groups) } // resolveValues iterates through the attributes, resolving them and putting them into the map. // If a group is encountered (as an attribute), it will be separately resolved and added as a subtree. // Since attributes are ordered from oldest to newest, it overwrites keys as it goes. -func (h *OverwriteHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, depth int) { +func (h *OverwriteHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, groups []string) { var ok bool for _, a := range attrs { a.Value = a.Value.Resolve() @@ -156,7 +157,7 @@ func (h *OverwriteHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog } // Default situation: resolve the key and put it into the map - a.Key, ok = h.getKey(a.Key, depth) + a.Key, ok = h.resolveKey(groups, a.Key, 0) if !ok { continue } @@ -168,13 +169,13 @@ func (h *OverwriteHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog // Groups with empty keys are inlined if a.Key == "" { - h.resolveValues(uniq, a.Value.Group(), depth) + h.resolveValues(uniq, a.Value.Group(), groups) continue } // Create a subtree for this group uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.resolveValues(uniqGroup, a.Value.Group(), depth+1) + h.resolveValues(uniqGroup, a.Value.Group(), append(slices.Clip(groups), a.Key)) // Ignore empty groups, otherwise put subtree into the map if uniqGroup.Len() > 0 { From 26d9eb352991da88d8f99898500bd45b423e5847 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 03:58:39 -0600 Subject: [PATCH 03/11] Update helpers, append handler, ignore handler, with resolve key --- append_handler.go | 31 ++++++++++++++++--------------- helpers.go | 32 +++++++++----------------------- ignore_handler.go | 31 ++++++++++++++++--------------- 3 files changed, 41 insertions(+), 53 deletions(-) diff --git a/append_handler.go b/append_handler.go index 480ce4d..195c571 100644 --- a/append_handler.go +++ b/append_handler.go @@ -3,6 +3,7 @@ package slogdedup import ( "context" "log/slog" + "slices" "modernc.org/b/v2" ) @@ -15,7 +16,7 @@ type AppendHandlerOptions struct { // Function that will be called on all root level (not in a group) attribute keys. // Returns the new key value to use, and true to keep the attribute or false to drop it. // Can be used to drop, keep, or rename any attributes matching the builtin attributes. - ResolveBuiltinKeyConflict func(k string) (string, bool) + ResolveKey func(groups []string, key string, _ int) (string, bool) } // AppendHandler is a slog.Handler middleware that will deduplicate all attributes and @@ -25,7 +26,7 @@ type AppendHandler struct { next slog.Handler goa *groupOrAttrs keyCompare func(a, b string) int - getKey func(key string, depth int) (string, bool) + resolveKey func(groups []string, key string, _ int) (string, bool) } var _ slog.Handler = &AppendHandler{} // Assert conformance with interface @@ -59,14 +60,14 @@ func NewAppendHandler(next slog.Handler, opts *AppendHandlerOptions) *AppendHand if opts.KeyCompare == nil { opts.KeyCompare = CaseSensitiveCmp } - if opts.ResolveBuiltinKeyConflict == nil { - opts.ResolveBuiltinKeyConflict = IncrementIfBuiltinKeyConflict + if opts.ResolveKey == nil { + opts.ResolveKey = IncrementIfBuiltinKeyConflict } return &AppendHandler{ next: next, keyCompare: opts.KeyCompare, - getKey: getKeyClosure(opts.ResolveBuiltinKeyConflict), + resolveKey: opts.ResolveKey, } } @@ -89,7 +90,7 @@ func (h *AppendHandler) Handle(ctx context.Context, r slog.Record) error { // Resolve groups and with-attributes uniq := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniq, goas, 0) + h.createAttrTree(uniq, goas, nil) // Add all attributes to new record (because old record has all the old attributes) newR := &slog.Record{ @@ -121,16 +122,16 @@ func (h *AppendHandler) WithAttrs(attrs []slog.Attr) slog.Handler { // createAttrTree recursively goes through all groupOrAttrs, resolving their attributes and creating subtrees as // necessary, adding the results to the map -func (h *AppendHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, depth int) { +func (h *AppendHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, groups []string) { if len(goas) == 0 { return } // If a group is encountered, create a subtree for that group and all groupOrAttrs after it if goas[0].group != "" { - if key, keep := h.getKey(goas[0].group, depth); keep { + if key, keep := h.resolveKey(groups, goas[0].group, 0); keep { uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniqGroup, goas[1:], depth+1) + h.createAttrTree(uniqGroup, goas[1:], append(slices.Clip(groups), key)) // Ignore empty groups, otherwise put subtree into the map if uniqGroup.Len() > 0 { // Put calls func(oldValue, true) if key already exists, or func(oldValue, false) if it doesn't. @@ -151,15 +152,15 @@ func (h *AppendHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupO } // Otherwise, set all attributes for this groupOrAttrs, and then call again for remaining groupOrAttrs's - h.resolveValues(uniq, goas[0].attrs, depth) - h.createAttrTree(uniq, goas[1:], depth) + h.resolveValues(uniq, goas[0].attrs, groups) + h.createAttrTree(uniq, goas[1:], groups) } // resolveValues iterates through the attributes, resolving them and putting them into the map. // If a group is encountered (as an attribute), it will be separately resolved and added as a subtree. // Since attributes are ordered from oldest to newest, it creates a slice whenever it detects the key already exists, // appending the new attribute, then overwriting the key with that slice. -func (h *AppendHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, depth int) { +func (h *AppendHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, groups []string) { var keep bool for _, a := range attrs { a.Value = a.Value.Resolve() @@ -168,7 +169,7 @@ func (h *AppendHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.At } // Default situation: resolve the key and put it into the map - a.Key, keep = h.getKey(a.Key, depth) + a.Key, keep = h.resolveKey(groups, a.Key, 0) if !keep { continue } @@ -189,13 +190,13 @@ func (h *AppendHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.At // Groups with empty keys are inlined if a.Key == "" { - h.resolveValues(uniq, a.Value.Group(), depth) + h.resolveValues(uniq, a.Value.Group(), groups) continue } // Create a subtree for this group uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.resolveValues(uniqGroup, a.Value.Group(), depth+1) + h.resolveValues(uniqGroup, a.Value.Group(), append(slices.Clip(groups), a.Key)) // Ignore empty groups, otherwise put subtree into the map if uniqGroup.Len() > 0 { diff --git a/helpers.go b/helpers.go index 17d3dda..ad27d4a 100644 --- a/helpers.go +++ b/helpers.go @@ -8,42 +8,28 @@ import ( "modernc.org/b/v2" ) -// TODO: also create a sorting middleware as well -// TODO: also create a pretty json printer that still prints to only 1 line, just prettier - -// getKeyClosure returns a function to be used to resolve a key at the root level, determining its behavior when it -// would otherwise conflict/duplicate the 4 built-in attribute keys (time, level, msg, source). -func getKeyClosure(resolveBuiltinKeyConflict func(k string) (string, bool)) func(key string, depth int) (string, bool) { - return func(key string, depth int) (string, bool) { - if depth == 0 { - return resolveBuiltinKeyConflict(key) - } - return key, true - } -} - // IncrementIfBuiltinKeyConflict will, if there is a conflict/duplication at the root level (not in a group) with one of // the built-in keys, add "#01" to the end of the key -func IncrementIfBuiltinKeyConflict(key string) (string, bool) { - if DoesBuiltinKeyConflict(key) { - return IncrementKeyName(key, 1), true // Don't overwrite the built-in attribute keys +func IncrementIfBuiltinKeyConflict(groups []string, key string, index int) (string, bool) { + if len(groups) == 0 && DoesBuiltinKeyConflict(key) { + return IncrementKeyName(key, index+1), true // Don't overwrite the built-in attribute keys } - return key, true + return IncrementKeyName(key, index), true } // DropIfBuiltinKeyConflict will, if there is a conflict/duplication at the root level (not in a group) with one of the // built-in keys, drop the whole attribute -func DropIfBuiltinKeyConflict(key string) (string, bool) { - if DoesBuiltinKeyConflict(key) { +func DropIfBuiltinKeyConflict(groups []string, key string, index int) (string, bool) { + if len(groups) == 0 && DoesBuiltinKeyConflict(key) { return "", false // Drop the attribute } - return key, true + return IncrementKeyName(key, index), true } // KeepIfBuiltinKeyConflict will keep all keys even if there would be a conflict/duplication at the root level (not in a // group) with one of the built-in keys -func KeepIfBuiltinKeyConflict(key string) (string, bool) { - return key, true // Keep all +func KeepIfBuiltinKeyConflict(_ []string, key string, index int) (string, bool) { + return IncrementKeyName(key, index), true // Keep all } // DoesBuiltinKeyConflict returns true if the key conflicts with the builtin keys. diff --git a/ignore_handler.go b/ignore_handler.go index 626ffcb..37de6e1 100644 --- a/ignore_handler.go +++ b/ignore_handler.go @@ -3,6 +3,7 @@ package slogdedup import ( "context" "log/slog" + "slices" "modernc.org/b/v2" ) @@ -15,7 +16,7 @@ type IgnoreHandlerOptions struct { // Function that will be called on all root level (not in a group) attribute keys. // Returns the new key value to use, and true to keep the attribute or false to drop it. // Can be used to drop, keep, or rename any attributes matching the builtin attributes. - ResolveBuiltinKeyConflict func(k string) (string, bool) + ResolveKey func(groups []string, key string, _ int) (string, bool) } // IgnoreHandler is a slog.Handler middleware that will deduplicate all attributes and @@ -25,7 +26,7 @@ type IgnoreHandler struct { next slog.Handler goa *groupOrAttrs keyCompare func(a, b string) int - getKey func(key string, depth int) (string, bool) + resolveKey func(groups []string, key string, _ int) (string, bool) } var _ slog.Handler = &IgnoreHandler{} // Assert conformance with interface @@ -59,14 +60,14 @@ func NewIgnoreHandler(next slog.Handler, opts *IgnoreHandlerOptions) *IgnoreHand if opts.KeyCompare == nil { opts.KeyCompare = CaseSensitiveCmp } - if opts.ResolveBuiltinKeyConflict == nil { - opts.ResolveBuiltinKeyConflict = IncrementIfBuiltinKeyConflict + if opts.ResolveKey == nil { + opts.ResolveKey = IncrementIfBuiltinKeyConflict } return &IgnoreHandler{ next: next, keyCompare: opts.KeyCompare, - getKey: getKeyClosure(opts.ResolveBuiltinKeyConflict), + resolveKey: opts.ResolveKey, } } @@ -89,7 +90,7 @@ func (h *IgnoreHandler) Handle(ctx context.Context, r slog.Record) error { // Resolve groups and with-attributes uniq := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniq, goas, 0) + h.createAttrTree(uniq, goas, nil) // Add all attributes to new record (because old record has all the old attributes) newR := &slog.Record{ @@ -121,16 +122,16 @@ func (h *IgnoreHandler) WithAttrs(attrs []slog.Attr) slog.Handler { // createAttrTree recursively goes through all groupOrAttrs, resolving their attributes and creating subtrees as // necessary, adding the results to the map -func (h *IgnoreHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, depth int) { +func (h *IgnoreHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupOrAttrs, groups []string) { if len(goas) == 0 { return } // If a group is encountered, create a subtree for that group and all groupOrAttrs after it if goas[0].group != "" { - if key, ok := h.getKey(goas[0].group, depth); ok { + if key, ok := h.resolveKey(groups, goas[0].group, 0); ok { uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.createAttrTree(uniqGroup, goas[1:], depth+1) + h.createAttrTree(uniqGroup, goas[1:], append(slices.Clip(groups), key)) // Ignore empty groups, otherwise put subtree into the map if uniqGroup.Len() > 0 { // Put calls func(oldValue, true) if key already exists, or func(oldValue, false) if it doesn't. @@ -147,14 +148,14 @@ func (h *IgnoreHandler) createAttrTree(uniq *b.Tree[string, any], goas []*groupO } // Otherwise, set all attributes for this groupOrAttrs, and then call again for remaining groupOrAttrs's - h.resolveValues(uniq, goas[0].attrs, depth) - h.createAttrTree(uniq, goas[1:], depth) + h.resolveValues(uniq, goas[0].attrs, groups) + h.createAttrTree(uniq, goas[1:], groups) } // resolveValues iterates through the attributes, resolving them and putting them into the map. // If a group is encountered (as an attribute), it will be separately resolved and added as a subtree. // Since attributes are ordered from oldest to newest, it ignores keys if they already exist. -func (h *IgnoreHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, depth int) { +func (h *IgnoreHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.Attr, groups []string) { var ok bool for _, a := range attrs { a.Value = a.Value.Resolve() @@ -163,7 +164,7 @@ func (h *IgnoreHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.At } // Default situation: resolve the key and put it into the map - a.Key, ok = h.getKey(a.Key, depth) + a.Key, ok = h.resolveKey(groups, a.Key, 0) if !ok { continue } @@ -180,13 +181,13 @@ func (h *IgnoreHandler) resolveValues(uniq *b.Tree[string, any], attrs []slog.At // Groups with empty keys are inlined if a.Key == "" { - h.resolveValues(uniq, a.Value.Group(), depth) + h.resolveValues(uniq, a.Value.Group(), groups) continue } // Create a subtree for this group uniqGroup := b.TreeNew[string, any](h.keyCompare) - h.resolveValues(uniqGroup, a.Value.Group(), depth+1) + h.resolveValues(uniqGroup, a.Value.Group(), append(slices.Clip(groups), a.Key)) // Ignore empty groups, otherwise put subtree into the map if uniqGroup.Len() > 0 { From c4bd7bc14abe7e62c29268446df9752aa378611a Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 03:59:05 -0600 Subject: [PATCH 04/11] Update tests --- append_handler_test.go | 166 ++++++++++++++++++++------------------ helpers_test.go | 1 + ignore_handler_test.go | 43 +++++----- increment_handler_test.go | 134 +++++++++++++++--------------- overwrite_handler_test.go | 88 ++++++++++---------- 5 files changed, 226 insertions(+), 206 deletions(-) diff --git a/append_handler_test.go b/append_handler_test.go index f49ff60..2ad9851 100644 --- a/append_handler_test.go +++ b/append_handler_test.go @@ -8,82 +8,90 @@ import ( /* { - "time": "2023-09-29T13:00:59Z", - "level": "INFO", - "msg": "main message", - "arg1": [ - "with1arg1", - "with2arg1" - ], - "arg2": "with1arg2", - "arg3": [ - "with1arg3", - "with2arg3" - ], - "arg4": "with2arg4", - "group1": [ - "with2group1", - { - "arg1": [ - "group1with3arg1", - "group1with4arg1", - "main1arg1" - ], - "arg2": "group1with3arg2", - "arg3": [ - "group1with3arg3", - "group1with4arg3" - ], - "arg4": "group1with4arg4", - "arg5": "with4inlinedGroupArg5", - "arg6": "main1arg6", - "level": [ - "with4overwritten", - "main1overwritten", - "main1level" - ], - "main1": "arg0", - "main1group3": { - "group3": [ - "group3overwritten", - "group3arg0" - ] - }, - "msg": "with4msg", - "overwrittenGroup": [ - { - "arg": "arg" - }, - "with4overwrittenGroup" - ], - "separateGroup2": { - "arg1": "group2arg1", - "arg2": "group2arg2", - "group2": "group2arg0" - }, - "source": "with3source", - "time": "with3time", - "with3": "arg0", - "with4": "arg0" - } - ], - "level#01": "with2level", - "msg#01": [ - "prexisting01", - "with2msg", - "with2msg2" - ], - "msg#01a": "seekbug01a", - "msg#02": "seekbug02", - "source#01": "with1source", - "time#01": "with1time", - "typed": [ - "overwritten", - 3, - true - ], - "with1": "arg0", - "with2": "arg0" + "time": "2023-09-29T13:00:59Z", + "level": "INFO", + "msg": "main message", + "arg1": [ + "with1arg1", + "with2arg1" + ], + "arg2": "with1arg2", + "arg3": [ + "with1arg3", + "with2arg3" + ], + "arg4": "with2arg4", + "group1": [ + "with2group1", + { + "arg1": [ + "group1with3arg1", + "group1with4arg1", + "main1arg1" + ], + "arg2": "group1with3arg2", + "arg3": [ + "group1with3arg3", + "group1with4arg3" + ], + "arg4": "group1with4arg4", + "arg5": "with4inlinedGroupArg5", + "arg6": "main1arg6", + "level": [ + "with4overwritten", + "main1overwritten", + "main1level" + ], + "main1": "arg0", + "main1group3": { + "group3": [ + "group3overwritten", + "group3arg0" + ] + }, + "msg": "with4msg", + "overwrittenGroup": [ + { + "arg": "arg" + }, + "with4overwrittenGroup" + ], + "separateGroup2": { + "arg1": "group2arg1", + "arg2": "group2arg2", + "group2": "group2arg0" + }, + "source": "with3source", + "time": "with3time", + "with3": "arg0", + "with4": "arg0" + } + ], + "level#01": [ + "with2level", + { + "levelGroupKey": "levelGroupValue" + }, + { + "inlinedLevelGroupKey": "inlinedLevelGroupValue" + } + ], + "msg#01": [ + "prexisting01", + "with2msg", + "with2msg2" + ], + "msg#01a": "seekbug01a", + "msg#02": "seekbug02", + "source#01": "with1source", + "time#01": "with1time", + "typed": [ + "overwritten", + 3, + true + ], + "with1": "arg0", + "with2": "arg0" } */ func TestAppendHandler(t *testing.T) { @@ -100,7 +108,7 @@ func TestAppendHandler(t *testing.T) { } jStr := strings.TrimSpace(string(jBytes)) - expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":["with1arg1","with2arg1"],"arg2":"with1arg2","arg3":["with1arg3","with2arg3"],"arg4":"with2arg4","group1":["with2group1",{"arg1":["group1with3arg1","group1with4arg1","main1arg1"],"arg2":"group1with3arg2","arg3":["group1with3arg3","group1with4arg3"],"arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":["with4overwritten","main1overwritten","main1level"],"main1":"arg0","main1group3":{"group3":["group3overwritten","group3arg0"]},"msg":"with4msg","overwrittenGroup":[{"arg":"arg"},"with4overwrittenGroup"],"separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"}],"level#01":"with2level","msg#01":["prexisting01","with2msg","with2msg2"],"msg#01a":"seekbug01a","msg#02":"seekbug02","source#01":"with1source","time#01":"with1time","typed":["overwritten",3,true],"with1":"arg0","with2":"arg0"}` + expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":["with1arg1","with2arg1"],"arg2":"with1arg2","arg3":["with1arg3","with2arg3"],"arg4":"with2arg4","group1":["with2group1",{"arg1":["group1with3arg1","group1with4arg1","main1arg1"],"arg2":"group1with3arg2","arg3":["group1with3arg3","group1with4arg3"],"arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":["with4overwritten","main1overwritten","main1level"],"main1":"arg0","main1group3":{"group3":["group3overwritten","group3arg0"]},"msg":"with4msg","overwrittenGroup":[{"arg":"arg"},"with4overwrittenGroup"],"separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"}],"level#01":["with2level",{"levelGroupKey":"levelGroupValue"},{"inlinedLevelGroupKey":"inlinedLevelGroupValue"}],"msg#01":["prexisting01","with2msg","with2msg2"],"msg#01a":"seekbug01a","msg#02":"seekbug02","source#01":"with1source","time#01":"with1time","typed":["overwritten",3,true],"with1":"arg0","with2":"arg0"}` if jStr != expected { t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) } @@ -126,8 +134,8 @@ func TestAppendHandler_CaseInsensitiveKeepIfBuiltinConflict(t *testing.T) { tester := &testHandler{} h := NewAppendMiddleware(&AppendHandlerOptions{ - KeyCompare: CaseInsensitiveCmp, - ResolveBuiltinKeyConflict: KeepIfBuiltinKeyConflict, + KeyCompare: CaseInsensitiveCmp, + ResolveKey: KeepIfBuiltinKeyConflict, })(tester) log := slog.New(h) diff --git a/helpers_test.go b/helpers_test.go index d8b3ffc..fac0096 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -77,6 +77,7 @@ func logComplex(t *testing.T, handler slog.Handler) { log = log.With("with1", "arg0", "arg1", "with1arg1", "arg2", "with1arg2", "arg3", "with1arg3", slog.SourceKey, "with1source", slog.TimeKey, "with1time", slog.Group("emptyGroup"), "typed", "overwritten", slog.Int("typed", 3)) log = log.With("with2", "arg0", "arg1", "with2arg1", "arg3", "with2arg3", "arg4", "with2arg4", "msg#01", "prexisting01", "msg#01a", "seekbug01a", "msg#02", "seekbug02", slog.MessageKey, "with2msg", slog.MessageKey, "with2msg2", slog.LevelKey, "with2level", "group1", "with2group1", slog.Bool("typed", true)) + log = log.With(slog.Group(slog.LevelKey, "levelGroupKey", "levelGroupValue"), slog.Group("", slog.Group(slog.LevelKey, "inlinedLevelGroupKey", "inlinedLevelGroupValue"))) log = log.WithGroup("group1").With(slog.Attr{}) log = log.With("with3", "arg0", "arg1", "group1with3arg1", "arg2", "group1with3arg2", "arg3", "group1with3arg3", slog.Group("overwrittenGroup", "arg", "arg"), slog.Group("separateGroup2", "group2", "group2arg0", "arg1", "group2arg1", "arg2", "group2arg2"), slog.SourceKey, "with3source", slog.TimeKey, "with3time") log = log.WithGroup("").WithGroup("") diff --git a/ignore_handler_test.go b/ignore_handler_test.go index 7482374..216175c 100644 --- a/ignore_handler_test.go +++ b/ignore_handler_test.go @@ -8,23 +8,23 @@ import ( /* { - "time": "2023-09-29T13:00:59Z", - "level": "INFO", - "msg": "main message", - "arg1": "with1arg1", - "arg2": "with1arg2", - "arg3": "with1arg3", - "arg4": "with2arg4", - "group1": "with2group1", - "level#01": "with2level", - "msg#01": "prexisting01", - "msg#01a": "seekbug01a", - "msg#02": "seekbug02", - "source#01": "with1source", - "time#01": "with1time", - "typed": "overwritten", - "with1": "arg0", - "with2": "arg0" + "time": "2023-09-29T13:00:59Z", + "level": "INFO", + "msg": "main message", + "arg1": "with1arg1", + "arg2": "with1arg2", + "arg3": "with1arg3", + "arg4": "with2arg4", + "group1": "with2group1", + "level#01": "with2level", + "msg#01": "prexisting01", + "msg#01a": "seekbug01a", + "msg#02": "seekbug02", + "source#01": "with1source", + "time#01": "with1time", + "typed": "overwritten", + "with1": "arg0", + "with2": "arg0" } */ func TestIgnoreHandler(t *testing.T) { @@ -58,11 +58,14 @@ func TestIgnoreHandler_ResolveBuiltinKeyConflict(t *testing.T) { tester := &testHandler{} h := NewIgnoreMiddleware(&IgnoreHandlerOptions{ - ResolveBuiltinKeyConflict: func(k string) (string, bool) { - if k == "time" { + ResolveKey: func(groups []string, key string, _ int) (string, bool) { + if len(groups) > 0 { + return key, true + } + if key == "time" { return "", false } else { - return "arg-" + k, true + return "arg-" + key, true } }, })(tester) diff --git a/increment_handler_test.go b/increment_handler_test.go index 866836e..fb89615 100644 --- a/increment_handler_test.go +++ b/increment_handler_test.go @@ -9,62 +9,68 @@ import ( /* { - "time": "2023-09-29T13:00:59Z", - "level": "INFO", - "msg": "main message", - "arg1": "with1arg1", - "arg1#01": "with2arg1", - "arg2": "with1arg2", - "arg3": "with1arg3", - "arg3#01": "with2arg3", - "arg4": "with2arg4", - "group1": "with2group1", - "group1#01": { - "arg1": "group1with3arg1", - "arg1#01": "group1with4arg1", - "arg1#02": "main1arg1", - "arg2": "group1with3arg2", - "arg3": "group1with3arg3", - "arg3#01": "group1with4arg3", - "arg4": "group1with4arg4", - "arg5": "with4inlinedGroupArg5", - "arg6": "main1arg6", - "level": "with4overwritten", - "level#01": "main1overwritten", - "level#02": "main1level", - "main1": "arg0", - "main1group3": { - "group3": "group3overwritten", - "group3#01": "group3arg0" - }, - "msg": "with4msg", - "overwrittenGroup": { - "arg": "arg" - }, - "overwrittenGroup#01": "with4overwrittenGroup", - "separateGroup2": { - "arg1": "group2arg1", - "arg2": "group2arg2", - "group2": "group2arg0" - }, - "source": "with3source", - "time": "with3time", - "with3": "arg0", - "with4": "arg0" - }, - "level#01": "with2level", - "msg#01": "prexisting01", - "msg#01a": "seekbug01a", - "msg#02": "seekbug02", - "msg#03": "with2msg", - "msg#04": "with2msg2", - "source#01": "with1source", - "time#01": "with1time", - "typed": "overwritten", - "typed#01": 3, - "typed#02": true, - "with1": "arg0", - "with2": "arg0" + "time": "2023-09-29T13:00:59Z", + "level": "INFO", + "msg": "main message", + "arg1": "with1arg1", + "arg1#01": "with2arg1", + "arg2": "with1arg2", + "arg3": "with1arg3", + "arg3#01": "with2arg3", + "arg4": "with2arg4", + "group1": "with2group1", + "group1#01": { + "arg1": "group1with3arg1", + "arg1#01": "group1with4arg1", + "arg1#02": "main1arg1", + "arg2": "group1with3arg2", + "arg3": "group1with3arg3", + "arg3#01": "group1with4arg3", + "arg4": "group1with4arg4", + "arg5": "with4inlinedGroupArg5", + "arg6": "main1arg6", + "level": "with4overwritten", + "level#01": "main1overwritten", + "level#02": "main1level", + "main1": "arg0", + "main1group3": { + "group3": "group3overwritten", + "group3#01": "group3arg0" + }, + "msg": "with4msg", + "overwrittenGroup": { + "arg": "arg" + }, + "overwrittenGroup#01": "with4overwrittenGroup", + "separateGroup2": { + "arg1": "group2arg1", + "arg2": "group2arg2", + "group2": "group2arg0" + }, + "source": "with3source", + "time": "with3time", + "with3": "arg0", + "with4": "arg0" + }, + "level#01": "with2level", + "level#02": { + "levelGroupKey": "levelGroupValue" + }, + "level#03": { + "inlinedLevelGroupKey": "inlinedLevelGroupValue" + }, + "msg#01": "prexisting01", + "msg#01a": "seekbug01a", + "msg#02": "seekbug02", + "msg#03": "with2msg", + "msg#04": "with2msg2", + "source#01": "with1source", + "time#01": "with1time", + "typed": "overwritten", + "typed#01": 3, + "typed#02": true, + "with1": "arg0", + "with2": "arg0" } */ func TestIncrementHandler(t *testing.T) { @@ -81,7 +87,7 @@ func TestIncrementHandler(t *testing.T) { } jStr := strings.TrimSpace(string(jBytes)) - expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":"with1arg1","arg1#01":"with2arg1","arg2":"with1arg2","arg3":"with1arg3","arg3#01":"with2arg3","arg4":"with2arg4","group1":"with2group1","group1#01":{"arg1":"group1with3arg1","arg1#01":"group1with4arg1","arg1#02":"main1arg1","arg2":"group1with3arg2","arg3":"group1with3arg3","arg3#01":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"with4overwritten","level#01":"main1overwritten","level#02":"main1level","main1":"arg0","main1group3":{"group3":"group3overwritten","group3#01":"group3arg0"},"msg":"with4msg","overwrittenGroup":{"arg":"arg"},"overwrittenGroup#01":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":"with2level","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","msg#03":"with2msg","msg#04":"with2msg2","source#01":"with1source","time#01":"with1time","typed":"overwritten","typed#01":3,"typed#02":true,"with1":"arg0","with2":"arg0"}` + expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":"with1arg1","arg1#01":"with2arg1","arg2":"with1arg2","arg3":"with1arg3","arg3#01":"with2arg3","arg4":"with2arg4","group1":"with2group1","group1#01":{"arg1":"group1with3arg1","arg1#01":"group1with4arg1","arg1#02":"main1arg1","arg2":"group1with3arg2","arg3":"group1with3arg3","arg3#01":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"with4overwritten","level#01":"main1overwritten","level#02":"main1level","main1":"arg0","main1group3":{"group3":"group3overwritten","group3#01":"group3arg0"},"msg":"with4msg","overwrittenGroup":{"arg":"arg"},"overwrittenGroup#01":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":"with2level","level#02":{"levelGroupKey":"levelGroupValue"},"level#03":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","msg#03":"with2msg","msg#04":"with2msg2","source#01":"with1source","time#01":"with1time","typed":"overwritten","typed#01":3,"typed#02":true,"with1":"arg0","with2":"arg0"}` if jStr != expected { t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) } @@ -98,18 +104,18 @@ func TestIncrementHandler_DoesKeyConflict_IncrementKeyName(t *testing.T) { tester := &testHandler{} h := NewIncrementMiddleware(&IncrementHandlerOptions{ - DoesBuiltinKeyConflict: func(key string) bool { + ResolveKey: func(groups []string, key string, index int) (string, bool) { if key == "foo" { - return true + return key + "@" + strconv.Itoa(index+1), true } - return false - }, - IncrementKeyName: func(key string, index int) string { - return key + "@" + strconv.Itoa(index) + if key == "hello" { + return "", false + } + return key, true }, })(tester) - slog.New(h).Info("main message", "foo", "bar") + slog.New(h).Info("main message", "foo", "bar", "hello", "world") jBytes, err := tester.MarshalJSON() if err != nil { diff --git a/overwrite_handler_test.go b/overwrite_handler_test.go index 5db4e16..4e0454b 100644 --- a/overwrite_handler_test.go +++ b/overwrite_handler_test.go @@ -8,46 +8,48 @@ import ( /* { - "time": "2023-09-29T13:00:59Z", - "level": "INFO", - "msg": "main message", - "arg1": "with2arg1", - "arg2": "with1arg2", - "arg3": "with2arg3", - "arg4": "with2arg4", - "group1": { - "arg1": "main1arg1", - "arg2": "group1with3arg2", - "arg3": "group1with4arg3", - "arg4": "group1with4arg4", - "arg5": "with4inlinedGroupArg5", - "arg6": "main1arg6", - "level": "main1level", - "main1": "arg0", - "main1group3": { - "group3": "group3arg0" - }, - "msg": "with4msg", - "overwrittenGroup": "with4overwrittenGroup", - "separateGroup2": { - "arg1": "group2arg1", - "arg2": "group2arg2", - "group2": "group2arg0" - }, - "source": "with3source", - "time": "with3time", - "with3": "arg0", - "with4": "arg0" - }, - "level#01": "with2level", - "msg#01": "with2msg2", - "msg#01a": "seekbug01a", - "msg#02": "seekbug02", - "source#01": "with1source", - "time#01": "with1time", - "typed": true, - "with1": "arg0", - "with2": "arg0" + "time": "2023-09-29T13:00:59Z", + "level": "INFO", + "msg": "main message", + "arg1": "with2arg1", + "arg2": "with1arg2", + "arg3": "with2arg3", + "arg4": "with2arg4", + "group1": { + "arg1": "main1arg1", + "arg2": "group1with3arg2", + "arg3": "group1with4arg3", + "arg4": "group1with4arg4", + "arg5": "with4inlinedGroupArg5", + "arg6": "main1arg6", + "level": "main1level", + "main1": "arg0", + "main1group3": { + "group3": "group3arg0" + }, + "msg": "with4msg", + "overwrittenGroup": "with4overwrittenGroup", + "separateGroup2": { + "arg1": "group2arg1", + "arg2": "group2arg2", + "group2": "group2arg0" + }, + "source": "with3source", + "time": "with3time", + "with3": "arg0", + "with4": "arg0" + }, + "level#01": { + "inlinedLevelGroupKey": "inlinedLevelGroupValue" + }, + "msg#01": "with2msg2", + "msg#01a": "seekbug01a", + "msg#02": "seekbug02", + "source#01": "with1source", + "time#01": "with1time", + "typed": true, + "with1": "arg0", + "with2": "arg0" } */ func TestOverwriteHandler(t *testing.T) { @@ -64,7 +66,7 @@ func TestOverwriteHandler(t *testing.T) { } jStr := strings.TrimSpace(string(jBytes)) - expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":"with2arg1","arg2":"with1arg2","arg3":"with2arg3","arg4":"with2arg4","group1":{"arg1":"main1arg1","arg2":"group1with3arg2","arg3":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"main1level","main1":"arg0","main1group3":{"group3":"group3arg0"},"msg":"with4msg","overwrittenGroup":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":"with2level","msg#01":"with2msg2","msg#01a":"seekbug01a","msg#02":"seekbug02","source#01":"with1source","time#01":"with1time","typed":true,"with1":"arg0","with2":"arg0"}` + expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":"with2arg1","arg2":"with1arg2","arg3":"with2arg3","arg4":"with2arg4","group1":{"arg1":"main1arg1","arg2":"group1with3arg2","arg3":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"main1level","main1":"arg0","main1group3":{"group3":"group3arg0"},"msg":"with4msg","overwrittenGroup":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"msg#01":"with2msg2","msg#01a":"seekbug01a","msg#02":"seekbug02","source#01":"with1source","time#01":"with1time","typed":true,"with1":"arg0","with2":"arg0"}` if jStr != expected { t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) } @@ -89,8 +91,8 @@ func TestOverwriteHandler_CaseInsensitiveDropBuiltinConflicts(t *testing.T) { tester := &testHandler{} h := NewOverwriteMiddleware(&OverwriteHandlerOptions{ - KeyCompare: CaseInsensitiveCmp, - ResolveBuiltinKeyConflict: DropIfBuiltinKeyConflict, + KeyCompare: CaseInsensitiveCmp, + ResolveKey: DropIfBuiltinKeyConflict, })(tester) log := slog.New(h) From 620a8163c8dadc5f569eadaaa4277c9ec9a425f2 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 08:28:35 -0600 Subject: [PATCH 05/11] Add more test cases --- append_handler_test.go | 11 +++++++++-- helpers_test.go | 9 +++++++-- ignore_handler_test.go | 11 +++++++++-- increment_handler_test.go | 11 +++++++++-- overwrite_handler_test.go | 11 +++++++++-- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/append_handler_test.go b/append_handler_test.go index 2ad9851..c96006b 100644 --- a/append_handler_test.go +++ b/append_handler_test.go @@ -9,7 +9,7 @@ import ( /* { "time": "2023-09-29T13:00:59Z", - "level": "INFO", + "level": "WARN", "msg": "main message", "arg1": [ "with1arg1", @@ -76,6 +76,9 @@ import ( "inlinedLevelGroupKey": "inlinedLevelGroupValue" } ], + "logging.googleapis.com/sourceLocation": "sourceLocationArg", + "message": "messageArg", + "message#01": "message#01Arg", "msg#01": [ "prexisting01", "with2msg", @@ -83,8 +86,12 @@ import ( ], "msg#01a": "seekbug01a", "msg#02": "seekbug02", + "severity": "severityArg", "source#01": "with1source", + "sourceLoc": "sourceLocArg", "time#01": "with1time", + "timestamp": "timestampArg", + "timestampRenamed": "timestampRenamedArg", "typed": [ "overwritten", 3, @@ -108,7 +115,7 @@ func TestAppendHandler(t *testing.T) { } jStr := strings.TrimSpace(string(jBytes)) - expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":["with1arg1","with2arg1"],"arg2":"with1arg2","arg3":["with1arg3","with2arg3"],"arg4":"with2arg4","group1":["with2group1",{"arg1":["group1with3arg1","group1with4arg1","main1arg1"],"arg2":"group1with3arg2","arg3":["group1with3arg3","group1with4arg3"],"arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":["with4overwritten","main1overwritten","main1level"],"main1":"arg0","main1group3":{"group3":["group3overwritten","group3arg0"]},"msg":"with4msg","overwrittenGroup":[{"arg":"arg"},"with4overwrittenGroup"],"separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"}],"level#01":["with2level",{"levelGroupKey":"levelGroupValue"},{"inlinedLevelGroupKey":"inlinedLevelGroupValue"}],"msg#01":["prexisting01","with2msg","with2msg2"],"msg#01a":"seekbug01a","msg#02":"seekbug02","source#01":"with1source","time#01":"with1time","typed":["overwritten",3,true],"with1":"arg0","with2":"arg0"}` + expected := `{"time":"2023-09-29T13:00:59Z","level":"WARN","msg":"main message","arg1":["with1arg1","with2arg1"],"arg2":"with1arg2","arg3":["with1arg3","with2arg3"],"arg4":"with2arg4","group1":["with2group1",{"arg1":["group1with3arg1","group1with4arg1","main1arg1"],"arg2":"group1with3arg2","arg3":["group1with3arg3","group1with4arg3"],"arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":["with4overwritten","main1overwritten","main1level"],"main1":"arg0","main1group3":{"group3":["group3overwritten","group3arg0"]},"msg":"with4msg","overwrittenGroup":[{"arg":"arg"},"with4overwrittenGroup"],"separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"}],"level#01":["with2level",{"levelGroupKey":"levelGroupValue"},{"inlinedLevelGroupKey":"inlinedLevelGroupValue"}],"logging.googleapis.com/sourceLocation":"sourceLocationArg","message":"messageArg","message#01":"message#01Arg","msg#01":["prexisting01","with2msg","with2msg2"],"msg#01a":"seekbug01a","msg#02":"seekbug02","severity":"severityArg","source#01":"with1source","sourceLoc":"sourceLocArg","time#01":"with1time","timestamp":"timestampArg","timestampRenamed":"timestampRenamedArg","typed":["overwritten",3,true],"with1":"arg0","with2":"arg0"}` if jStr != expected { t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) } diff --git a/helpers_test.go b/helpers_test.go index fac0096..bda4a8d 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -77,12 +77,13 @@ func logComplex(t *testing.T, handler slog.Handler) { log = log.With("with1", "arg0", "arg1", "with1arg1", "arg2", "with1arg2", "arg3", "with1arg3", slog.SourceKey, "with1source", slog.TimeKey, "with1time", slog.Group("emptyGroup"), "typed", "overwritten", slog.Int("typed", 3)) log = log.With("with2", "arg0", "arg1", "with2arg1", "arg3", "with2arg3", "arg4", "with2arg4", "msg#01", "prexisting01", "msg#01a", "seekbug01a", "msg#02", "seekbug02", slog.MessageKey, "with2msg", slog.MessageKey, "with2msg2", slog.LevelKey, "with2level", "group1", "with2group1", slog.Bool("typed", true)) + log = log.With("timestamp", "timestampArg", "timestampRenamed", "timestampRenamedArg", "severity", "severityArg", "message", "messageArg", "message#01", "message#01Arg", "sourceLoc", "sourceLocArg", "logging.googleapis.com/sourceLocation", "sourceLocationArg") log = log.With(slog.Group(slog.LevelKey, "levelGroupKey", "levelGroupValue"), slog.Group("", slog.Group(slog.LevelKey, "inlinedLevelGroupKey", "inlinedLevelGroupValue"))) log = log.WithGroup("group1").With(slog.Attr{}) log = log.With("with3", "arg0", "arg1", "group1with3arg1", "arg2", "group1with3arg2", "arg3", "group1with3arg3", slog.Group("overwrittenGroup", "arg", "arg"), slog.Group("separateGroup2", "group2", "group2arg0", "arg1", "group2arg1", "arg2", "group2arg2"), slog.SourceKey, "with3source", slog.TimeKey, "with3time") log = log.WithGroup("").WithGroup("") log = log.With("with4", "arg0", "arg1", "group1with4arg1", "arg3", "group1with4arg3", "arg4", "group1with4arg4", slog.Group("", "arg5", "with4inlinedGroupArg5"), slog.String("overwrittenGroup", "with4overwrittenGroup"), slog.MessageKey, "with4msg", slog.LevelKey, "with4overwritten") - log.Info("main message", "main1", "arg0", "arg1", "main1arg1", "arg6", "main1arg6", slog.LevelKey, "main1overwritten", slog.LevelKey, "main1level", slog.Group("main1group3", "group3", "group3overwritten", "group3", "group3arg0")) + log.Warn("main message", "main1", "arg0", "arg1", "main1arg1", "arg6", "main1arg6", slog.LevelKey, "main1overwritten", slog.LevelKey, "main1level", slog.Group("main1group3", "group3", "group3overwritten", "group3", "group3arg0")) } type testHandler struct { @@ -120,13 +121,17 @@ func (h *testHandler) String() string { func (h *testHandler) MarshalJSON() ([]byte, error) { buf := &bytes.Buffer{} - err := slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}).Handle(context.Background(), h.Record) + err := h.MarshalWith(slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug})) if err != nil { return nil, err } return buf.Bytes(), nil } +func (h *testHandler) MarshalWith(handler slog.Handler) error { + return handler.Handle(context.Background(), h.Record) +} + func checkRecordForDuplicates(t *testing.T, r slog.Record) { t.Helper() diff --git a/ignore_handler_test.go b/ignore_handler_test.go index 216175c..df35e77 100644 --- a/ignore_handler_test.go +++ b/ignore_handler_test.go @@ -9,7 +9,7 @@ import ( /* { "time": "2023-09-29T13:00:59Z", - "level": "INFO", + "level": "WARN", "msg": "main message", "arg1": "with1arg1", "arg2": "with1arg2", @@ -17,11 +17,18 @@ import ( "arg4": "with2arg4", "group1": "with2group1", "level#01": "with2level", + "logging.googleapis.com/sourceLocation": "sourceLocationArg", + "message": "messageArg", + "message#01": "message#01Arg", "msg#01": "prexisting01", "msg#01a": "seekbug01a", "msg#02": "seekbug02", + "severity": "severityArg", "source#01": "with1source", + "sourceLoc": "sourceLocArg", "time#01": "with1time", + "timestamp": "timestampArg", + "timestampRenamed": "timestampRenamedArg", "typed": "overwritten", "with1": "arg0", "with2": "arg0" @@ -41,7 +48,7 @@ func TestIgnoreHandler(t *testing.T) { } jStr := strings.TrimSpace(string(jBytes)) - expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":"with1arg1","arg2":"with1arg2","arg3":"with1arg3","arg4":"with2arg4","group1":"with2group1","level#01":"with2level","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","source#01":"with1source","time#01":"with1time","typed":"overwritten","with1":"arg0","with2":"arg0"}` + expected := `{"time":"2023-09-29T13:00:59Z","level":"WARN","msg":"main message","arg1":"with1arg1","arg2":"with1arg2","arg3":"with1arg3","arg4":"with2arg4","group1":"with2group1","level#01":"with2level","logging.googleapis.com/sourceLocation":"sourceLocationArg","message":"messageArg","message#01":"message#01Arg","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","severity":"severityArg","source#01":"with1source","sourceLoc":"sourceLocArg","time#01":"with1time","timestamp":"timestampArg","timestampRenamed":"timestampRenamedArg","typed":"overwritten","with1":"arg0","with2":"arg0"}` if jStr != expected { t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) } diff --git a/increment_handler_test.go b/increment_handler_test.go index fb89615..a88c84e 100644 --- a/increment_handler_test.go +++ b/increment_handler_test.go @@ -10,7 +10,7 @@ import ( /* { "time": "2023-09-29T13:00:59Z", - "level": "INFO", + "level": "WARN", "msg": "main message", "arg1": "with1arg1", "arg1#01": "with2arg1", @@ -59,13 +59,20 @@ import ( "level#03": { "inlinedLevelGroupKey": "inlinedLevelGroupValue" }, + "logging.googleapis.com/sourceLocation": "sourceLocationArg", + "message": "messageArg", + "message#01": "message#01Arg", "msg#01": "prexisting01", "msg#01a": "seekbug01a", "msg#02": "seekbug02", "msg#03": "with2msg", "msg#04": "with2msg2", + "severity": "severityArg", "source#01": "with1source", + "sourceLoc": "sourceLocArg", "time#01": "with1time", + "timestamp": "timestampArg", + "timestampRenamed": "timestampRenamedArg", "typed": "overwritten", "typed#01": 3, "typed#02": true, @@ -87,7 +94,7 @@ func TestIncrementHandler(t *testing.T) { } jStr := strings.TrimSpace(string(jBytes)) - expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":"with1arg1","arg1#01":"with2arg1","arg2":"with1arg2","arg3":"with1arg3","arg3#01":"with2arg3","arg4":"with2arg4","group1":"with2group1","group1#01":{"arg1":"group1with3arg1","arg1#01":"group1with4arg1","arg1#02":"main1arg1","arg2":"group1with3arg2","arg3":"group1with3arg3","arg3#01":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"with4overwritten","level#01":"main1overwritten","level#02":"main1level","main1":"arg0","main1group3":{"group3":"group3overwritten","group3#01":"group3arg0"},"msg":"with4msg","overwrittenGroup":{"arg":"arg"},"overwrittenGroup#01":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":"with2level","level#02":{"levelGroupKey":"levelGroupValue"},"level#03":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","msg#03":"with2msg","msg#04":"with2msg2","source#01":"with1source","time#01":"with1time","typed":"overwritten","typed#01":3,"typed#02":true,"with1":"arg0","with2":"arg0"}` + expected := `{"time":"2023-09-29T13:00:59Z","level":"WARN","msg":"main message","arg1":"with1arg1","arg1#01":"with2arg1","arg2":"with1arg2","arg3":"with1arg3","arg3#01":"with2arg3","arg4":"with2arg4","group1":"with2group1","group1#01":{"arg1":"group1with3arg1","arg1#01":"group1with4arg1","arg1#02":"main1arg1","arg2":"group1with3arg2","arg3":"group1with3arg3","arg3#01":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"with4overwritten","level#01":"main1overwritten","level#02":"main1level","main1":"arg0","main1group3":{"group3":"group3overwritten","group3#01":"group3arg0"},"msg":"with4msg","overwrittenGroup":{"arg":"arg"},"overwrittenGroup#01":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":"with2level","level#02":{"levelGroupKey":"levelGroupValue"},"level#03":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"logging.googleapis.com/sourceLocation":"sourceLocationArg","message":"messageArg","message#01":"message#01Arg","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","msg#03":"with2msg","msg#04":"with2msg2","severity":"severityArg","source#01":"with1source","sourceLoc":"sourceLocArg","time#01":"with1time","timestamp":"timestampArg","timestampRenamed":"timestampRenamedArg","typed":"overwritten","typed#01":3,"typed#02":true,"with1":"arg0","with2":"arg0"}` if jStr != expected { t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) } diff --git a/overwrite_handler_test.go b/overwrite_handler_test.go index 4e0454b..a1bb281 100644 --- a/overwrite_handler_test.go +++ b/overwrite_handler_test.go @@ -9,7 +9,7 @@ import ( /* { "time": "2023-09-29T13:00:59Z", - "level": "INFO", + "level": "WARN", "msg": "main message", "arg1": "with2arg1", "arg2": "with1arg2", @@ -42,11 +42,18 @@ import ( "level#01": { "inlinedLevelGroupKey": "inlinedLevelGroupValue" }, + "logging.googleapis.com/sourceLocation": "sourceLocationArg", + "message": "messageArg", + "message#01": "message#01Arg", "msg#01": "with2msg2", "msg#01a": "seekbug01a", "msg#02": "seekbug02", + "severity": "severityArg", "source#01": "with1source", + "sourceLoc": "sourceLocArg", "time#01": "with1time", + "timestamp": "timestampArg", + "timestampRenamed": "timestampRenamedArg", "typed": true, "with1": "arg0", "with2": "arg0" @@ -66,7 +73,7 @@ func TestOverwriteHandler(t *testing.T) { } jStr := strings.TrimSpace(string(jBytes)) - expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"main message","arg1":"with2arg1","arg2":"with1arg2","arg3":"with2arg3","arg4":"with2arg4","group1":{"arg1":"main1arg1","arg2":"group1with3arg2","arg3":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"main1level","main1":"arg0","main1group3":{"group3":"group3arg0"},"msg":"with4msg","overwrittenGroup":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"msg#01":"with2msg2","msg#01a":"seekbug01a","msg#02":"seekbug02","source#01":"with1source","time#01":"with1time","typed":true,"with1":"arg0","with2":"arg0"}` + expected := `{"time":"2023-09-29T13:00:59Z","level":"WARN","msg":"main message","arg1":"with2arg1","arg2":"with1arg2","arg3":"with2arg3","arg4":"with2arg4","group1":{"arg1":"main1arg1","arg2":"group1with3arg2","arg3":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"main1level","main1":"arg0","main1group3":{"group3":"group3arg0"},"msg":"with4msg","overwrittenGroup":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"level#01":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"logging.googleapis.com/sourceLocation":"sourceLocationArg","message":"messageArg","message#01":"message#01Arg","msg#01":"with2msg2","msg#01a":"seekbug01a","msg#02":"seekbug02","severity":"severityArg","source#01":"with1source","sourceLoc":"sourceLocArg","time#01":"with1time","timestamp":"timestampArg","timestampRenamed":"timestampRenamedArg","typed":true,"with1":"arg0","with2":"arg0"}` if jStr != expected { t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) } From eaf014148c47958b6aaada6c522d98b03f866a3d Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 08:30:41 -0600 Subject: [PATCH 06/11] Create helper methods for resolving key and replacing attributes --- resolve_keys_replace_attrs.go | 130 ++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 resolve_keys_replace_attrs.go diff --git a/resolve_keys_replace_attrs.go b/resolve_keys_replace_attrs.go new file mode 100644 index 0000000..c2b371c --- /dev/null +++ b/resolve_keys_replace_attrs.go @@ -0,0 +1,130 @@ +package slogdedup + +import ( + "fmt" + "log/slog" + "strconv" +) + +// JoinResolveKey can be used to join together many slogdedup middlewares +// ...HandlerOptions.ResolveKey functions into a single one that applies all the +// rules in order. +func JoinResolveKey(resolveKeyFunctions ...func(groups []string, key string, index int) (string, bool)) func(groups []string, key string, index int) (string, bool) { + if len(resolveKeyFunctions) == 0 { + return nil + } + return func(groups []string, originalKey string, index int) (string, bool) { + var ok bool + key := originalKey + for _, f := range resolveKeyFunctions { + if key, ok = f(groups, key, index); !ok { + break + } + } + // Only increment once, and only if the key was not changed. + // This would happen if we have multiple duplicate keys in row. + if key != originalKey { + return key, ok + } + return IncrementKeyName(key, index), ok + } +} + +// JoinReplaceAttr can be used to join together many slog.HandlerOptions.ReplaceAttr +// into a single one that applies all rules in order. +func JoinReplaceAttr(replaceAttrFunctions ...func(groups []string, a slog.Attr) slog.Attr) func(groups []string, a slog.Attr) slog.Attr { + if len(replaceAttrFunctions) == 0 { + return nil + } + return func(groups []string, a slog.Attr) slog.Attr { + for _, f := range replaceAttrFunctions { + if a = f(groups, a); a.Equal(slog.Attr{}) { + break + } + } + return a + } +} + +// sink represents the final destination of the logs. +type sink struct { + // Only the keys that will be used for the builtins: + // (slog.TimeKey, slog.LevelKey, slog.MessageKey, slog.SourceKey) + builtins []string + + // Replacement key name and optional function to replace the value. + replacers map[string]attrReplacer +} + +// attrReplacer has the replacement key name, and optional function to replace the value +type attrReplacer struct { + key string + valuer func(v slog.Value) slog.Value +} + +// resolveKeys returns a closure that can be used with any slogdedup middlewares +// ...HandlerOptions.ResolveKey. Its purpose is to replace the key on any +// attributes or groups, except for the builtin attributes. Using replaceAttr on +// the final handler/sink is still required, in order to replace the builtin +// attribute keys. +func resolveKeys(dest sink) func(groups []string, key string, index int) (string, bool) { + // This function is for the dedup middlewares. + // These middlewares do not send the builtin's (time, level, msg, source), + // because they have no control over the keys that will be used. + // Only the final/sink handler knows what keys will be used. + // So avoid situations like "source", where we might have an added + // field already named "sourceLoc", and then later when the + // builtin "source" is logged by the sink it get replaced with + // "sourceLoc", ending up with duplicates. + // Example: slog.Info("main", slog.String(slog.MessageKey, "hello"), slog.String("message", "world")) + // Should, if using Graylog or Stackdriver, come out as: + // {"message":"main", "message#01":"hello", "message#02":"world"} + return func(groups []string, key string, index int) (string, bool) { + if len(groups) > 0 { + return key, true + } + + // Check replacers first + for oldKey, replacement := range dest.replacers { + if key == oldKey { + key = replacement.key + } + } + + // Check builtins last + for _, builtin := range dest.builtins { + if key == builtin { + return IncrementKeyName(key, index+1), true + } + } + return key, true + } +} + +// replaceAttr returns a closure that can be used with slog.HandlerOptions.ReplaceAttr. +// Its purpose is to replace the builtin keys and values only. +// All non-builtin attributes will have their keys modified by resolveKeys. +func replaceAttr(dest sink) func(groups []string, a slog.Attr) slog.Attr { + // This function is for the final handler (the sink). + // It knows what keys will be used for the builtin's (time, level, msg, source), + // and has the ability to modify those keys (and values) here. + // Standard library sinks, like slog.JSONHandler, do not have the ability to + // modify the groups at this point, hence why we are modifying them in the + // resolveKeys function on the dedup middleware instead. + return func(groups []string, a slog.Attr) slog.Attr { + if len(groups) > 0 { + return a + } + + for oldKey, replacement := range dest.replacers { + if a.Key == oldKey { + a.Key = replacement.key + if replacement.valuer != nil { + a.Value = replacement.valuer(a.Value) + } + return a + } + } + return a + } +} From 7ab7debde3bc279916ba85467baa71e7956a6a4f Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 08:35:42 -0600 Subject: [PATCH 07/11] Add replacers for Graylog --- resolve_keys_replace_attrs.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/resolve_keys_replace_attrs.go b/resolve_keys_replace_attrs.go index c2b371c..8fb1ffa 100644 --- a/resolve_keys_replace_attrs.go +++ b/resolve_keys_replace_attrs.go @@ -46,6 +46,37 @@ func JoinReplaceAttr(replaceAttrFunctions ...func(groups []string, a slog.Attr) } } +// ResolveKeyGraylog returns a ResolveKey function works for Graylog. +func ResolveKeyGraylog() func(groups []string, key string, index int) (string, bool) { + return resolveKeys(sinkGraylog) +} + +// ReplaceAttrGraylog returns a ReplaceAttr function works for Graylog. +func ReplaceAttrGraylog() func(groups []string, a slog.Attr) slog.Attr { + return replaceAttr(sinkGraylog) +} + +// Graylog https://graylog.org/ +var sinkGraylog = sink{ + builtins: []string{slog.TimeKey, slog.LevelKey, "message", "sourceLoc"}, + replacers: map[string]attrReplacer{ + // "timestamp" is the time of the record. Defaults to the time the log was received by grayload. + // If using a json extractor or rule, Graylog needs to have it set to a time object, not a string. + // So best to let your timestamp come in under a different key, then set it specifically with a pipeline rule. + "timestamp": {key: "timestampRenamed"}, + + // "message" is what Graylog will show when skimming. It defaults to the entire log payload. + // Have the builtin message use this as its key. + slog.MessageKey: {key: "message"}, + + // "source" is the IP address or similar of where the logs came from. + // Let Graylog keep its enchriched field, and rename our source location. + slog.SourceKey: {key: "sourceLoc"}, + }, +} + + + // sink represents the final destination of the logs. type sink struct { // Only the keys that will be used for the builtins: From a6e0f491787f87e2024c0498178d77d3119f7ec1 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 08:40:58 -0600 Subject: [PATCH 08/11] Add replacers for Stackdriver (GCP) --- resolve_keys_replace_attrs.go | 77 +++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/resolve_keys_replace_attrs.go b/resolve_keys_replace_attrs.go index 8fb1ffa..56bdf17 100644 --- a/resolve_keys_replace_attrs.go +++ b/resolve_keys_replace_attrs.go @@ -76,6 +76,83 @@ var sinkGraylog = sink{ } +// ResolveKeyStackdriver returns a ResolveKey function works for Stackdriver +// (aka Google Cloud Operations, aka GCP Log Explorer). +func ResolveKeyStackdriver() func(groups []string, key string, index int) (string, bool) { + return resolveKeys(sinkStackdriver) +} + +// ReplaceAttrStackdriver returns a ReplaceAttr function works for Stackdriver +// (aka Google Cloud Operations, aka GCP Log Explorer). +func ReplaceAttrStackdriver() func(groups []string, a slog.Attr) slog.Attr { + return replaceAttr(sinkStackdriver) +} + +// Stackdriver, aka Google Cloud Operations, aka GCP Log Explorer +// https://cloud.google.com/products/operations +var sinkStackdriver = sink{ + builtins: []string{slog.TimeKey, "severity", "message", "logging.googleapis.com/sourceLocation"}, + replacers: map[string]attrReplacer{ + // The default slog time key is "time", which stackdriver will detect and parse: + // https://cloud.google.com/logging/docs/agent/logging/configuration#special-fields + + // "severity" is what Stackdriver uses for the log level: + // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity + // Have the builtin level use this as its key. + slog.LevelKey: {key: "severity", valuer: func(v slog.Value) slog.Value { + switch lvl := v.Any().(type) { + case slog.Level: + if lvl <= slog.LevelDebug { + return slog.StringValue("DEBUG") + } else if lvl <= slog.LevelInfo { + return slog.StringValue("INFO") + } else if lvl < slog.LevelWarn { + return slog.StringValue("NOTICE") + } else if lvl == slog.LevelWarn { + return slog.StringValue("WARNING") + } else if lvl <= slog.LevelError { + return slog.StringValue("ERROR") + } else if lvl <= slog.LevelError+4 { + return slog.StringValue("CRITICAL") + } else if lvl <= slog.LevelError+8 { + return slog.StringValue("ALERT") + } + return slog.StringValue("EMERGENCY") + default: + return v + } + }}, + + // "message" is what Stackdriver will show when skimming. It defaults to the entire log payload. + // Have the builtin message use this as its key. + slog.MessageKey: {key: "message"}, + + // "logging.googleapis.com/sourceLocation" is what Stackdriver expects for + // the key containing the file, line, and function values. + // Have the builtin source use this as its key. + slog.SourceKey: {key: "logging.googleapis.com/sourceLocation", valuer: func(v slog.Value) slog.Value { + // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogEntrySourceLocation + switch source := v.Any().(type) { + case *slog.Source: + if source == nil { + return v + } + return slog.AnyValue(struct { + Function string `json:"function"` + File string `json:"file"` + Line string `json:"line"` // slog.Source.Line is an int, GCP wants a string + }{ + Function: source.Function, + File: source.File, + Line: strconv.Itoa(source.Line), + }) + default: + fmt.Printf("SOURCE: %T: %#+v\n", source, source) + return v + } + }}, + }, +} // sink represents the final destination of the logs. type sink struct { From 297f7378e45420809db8460b9a50531155105b38 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 08:43:06 -0600 Subject: [PATCH 09/11] Add tests for replacers --- resolve_keys_replace_attrs_test.go | 76 ++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 resolve_keys_replace_attrs_test.go diff --git a/resolve_keys_replace_attrs_test.go b/resolve_keys_replace_attrs_test.go new file mode 100644 index 0000000..870b58e --- /dev/null +++ b/resolve_keys_replace_attrs_test.go @@ -0,0 +1,76 @@ +package slogdedup + +import ( + "bytes" + "log/slog" + "strings" + "testing" +) + +func TestFoo(t *testing.T) { + t.Parallel() + + resolvers := JoinResolveKey( + ResolveKeyStackdriver(), + ResolveKeyGraylog(), + ) + + replacers := JoinReplaceAttr( + func(groups []string, a slog.Attr) slog.Attr { + if len(groups) == 0 && a.Key == slog.SourceKey { + src := a.Value.Any().(*slog.Source) + src.File = "github.com/veqryn/slog-dedup/helpers_test.go" + src.Function = "github.com/veqryn/slog-dedup.logComplex" + src.Line = 85 + } + return a + }, + ReplaceAttrStackdriver(), + ReplaceAttrGraylog(), + ) + + tester := &testHandler{} + tests := []struct { + hander slog.Handler + expected string + }{ + { + hander: NewOverwriteHandler(tester, &OverwriteHandlerOptions{ResolveKey: resolvers}), + expected: `{"time":"2023-09-29T13:00:59Z","severity":"WARNING","logging.googleapis.com/sourceLocation":{"function":"github.com/veqryn/slog-dedup.logComplex","file":"github.com/veqryn/slog-dedup/helpers_test.go","line":"85"},"message":"main message","arg1":"with2arg1","arg2":"with1arg2","arg3":"with2arg3","arg4":"with2arg4","group1":{"arg1":"main1arg1","arg2":"group1with3arg2","arg3":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"main1level","main1":"arg0","main1group3":{"group3":"group3arg0"},"msg":"with4msg","overwrittenGroup":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"logging.googleapis.com/sourceLocation#01":"sourceLocationArg","message#01":"message#01Arg","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","severity#01":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"sourceLoc#01":"sourceLocArg","time#01":"with1time","timestampRenamed":"timestampRenamedArg","typed":true,"with1":"arg0","with2":"arg0"}`, + }, + { + hander: NewIgnoreHandler(tester, &IgnoreHandlerOptions{ResolveKey: resolvers}), + expected: `{"time":"2023-09-29T13:00:59Z","severity":"WARNING","logging.googleapis.com/sourceLocation":{"function":"github.com/veqryn/slog-dedup.logComplex","file":"github.com/veqryn/slog-dedup/helpers_test.go","line":"85"},"message":"main message","arg1":"with1arg1","arg2":"with1arg2","arg3":"with1arg3","arg4":"with2arg4","group1":"with2group1","logging.googleapis.com/sourceLocation#01":"with1source","message#01":"with2msg","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","severity#01":"with2level","sourceLoc#01":"sourceLocArg","time#01":"with1time","timestampRenamed":"timestampArg","typed":"overwritten","with1":"arg0","with2":"arg0"`, + }, + { + hander: NewAppendHandler(tester, &AppendHandlerOptions{ResolveKey: resolvers}), + expected: `{"time":"2023-09-29T13:00:59Z","severity":"WARNING","logging.googleapis.com/sourceLocation":{"function":"github.com/veqryn/slog-dedup.logComplex","file":"github.com/veqryn/slog-dedup/helpers_test.go","line":"85"},"message":"main message","arg1":["with1arg1","with2arg1"],"arg2":"with1arg2","arg3":["with1arg3","with2arg3"],"arg4":"with2arg4","group1":["with2group1",{"arg1":["group1with3arg1","group1with4arg1","main1arg1"],"arg2":"group1with3arg2","arg3":["group1with3arg3","group1with4arg3"],"arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":["with4overwritten","main1overwritten","main1level"],"main1":"arg0","main1group3":{"group3":["group3overwritten","group3arg0"]},"msg":"with4msg","overwrittenGroup":[{"arg":"arg"},"with4overwrittenGroup"],"separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"}],"logging.googleapis.com/sourceLocation#01":["with1source","sourceLocationArg"],"message#01":["with2msg","with2msg2","messageArg","message#01Arg"],"msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","severity#01":["with2level","severityArg",{"levelGroupKey":"levelGroupValue"},{"inlinedLevelGroupKey":"inlinedLevelGroupValue"}],"sourceLoc#01":"sourceLocArg","time#01":"with1time","timestampRenamed":["timestampArg","timestampRenamedArg"],"typed":["overwritten",3,true],"with1":"arg0","with2":"arg0"}`, + }, + { + hander: NewIncrementHandler(tester, &IncrementHandlerOptions{ResolveKey: resolvers}), + expected: `{"time":"2023-09-29T13:00:59Z","severity":"WARNING","logging.googleapis.com/sourceLocation":{"function":"github.com/veqryn/slog-dedup.logComplex","file":"github.com/veqryn/slog-dedup/helpers_test.go","line":"85"},"message":"main message","arg1":"with1arg1","arg1#01":"with2arg1","arg2":"with1arg2","arg3":"with1arg3","arg3#01":"with2arg3","arg4":"with2arg4","group1":"with2group1","group1#01":{"arg1":"group1with3arg1","arg1#01":"group1with4arg1","arg1#02":"main1arg1","arg2":"group1with3arg2","arg3":"group1with3arg3","arg3#01":"group1with4arg3","arg4":"group1with4arg4","arg5":"with4inlinedGroupArg5","arg6":"main1arg6","level":"with4overwritten","level#01":"main1overwritten","level#02":"main1level","main1":"arg0","main1group3":{"group3":"group3overwritten","group3#01":"group3arg0"},"msg":"with4msg","overwrittenGroup":{"arg":"arg"},"overwrittenGroup#01":"with4overwrittenGroup","separateGroup2":{"arg1":"group2arg1","arg2":"group2arg2","group2":"group2arg0"},"source":"with3source","time":"with3time","with3":"arg0","with4":"arg0"},"logging.googleapis.com/sourceLocation#01":"with1source","logging.googleapis.com/sourceLocation#02":"sourceLocationArg","message#01":"with2msg","message#01#01":"message#01Arg","message#02":"with2msg2","message#03":"messageArg","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","severity#01":"with2level","severity#02":"severityArg","severity#03":{"levelGroupKey":"levelGroupValue"},"severity#04":{"inlinedLevelGroupKey":"inlinedLevelGroupValue"},"sourceLoc#01":"sourceLocArg","time#01":"with1time","timestampRenamed":"timestampArg","timestampRenamed#01":"timestampRenamedArg","typed":"overwritten","typed#01":3,"typed#02":true,"with1":"arg0","with2":"arg0"}`, + }, + } + + for _, testCase := range tests { + logComplex(t, testCase.hander) + + buf := &bytes.Buffer{} + err := tester.MarshalWith(slog.NewJSONHandler(buf, &slog.HandlerOptions{AddSource: true, Level: slog.LevelDebug, ReplaceAttr: replacers})) + if err != nil { + t.Errorf("Unable to marshal json: %v", err) + continue + } + jStr := strings.TrimSpace(buf.String()) + + if jStr != testCase.expected { + t.Errorf("Expected:\n%s\nGot:\n%s", testCase.expected, jStr) + } + + // Uncomment to see the results + // t.Error(jStr) + // t.Error(tester.String()) + + checkRecordForDuplicates(t, tester.Record) + } +} From 411b08e520e487cede67f246564f5ae5d3676943 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 09:09:34 -0600 Subject: [PATCH 10/11] Update go docs --- append_handler.go | 13 ++++++++++--- helpers.go | 31 +++++++++++++++++-------------- ignore_handler.go | 13 ++++++++++--- increment_handler.go | 26 ++++++++++++++++---------- overwrite_handler.go | 13 ++++++++++--- resolve_keys_replace_attrs.go | 9 ++++----- 6 files changed, 67 insertions(+), 38 deletions(-) diff --git a/append_handler.go b/append_handler.go index 195c571..46d1e95 100644 --- a/append_handler.go +++ b/append_handler.go @@ -13,9 +13,16 @@ type AppendHandlerOptions struct { // Comparison function to determine if two keys are equal KeyCompare func(a, b string) int - // Function that will be called on all root level (not in a group) attribute keys. - // Returns the new key value to use, and true to keep the attribute or false to drop it. - // Can be used to drop, keep, or rename any attributes matching the builtin attributes. + // Function that will be called on each attribute and group, to determine + // the key to use. Returns the new key value to use, and true to keep the + // attribute or false to drop it. Can be used to drop, keep, or rename any + // attributes matching the builtin attributes. + // + // The first argument is a list of currently open groups that contain the + // Attr. It must not be retained or modified. + // + // ResolveKey will not be called for the built-in fields on slog.Record + // (ie: time, level, msg, and source). ResolveKey func(groups []string, key string, _ int) (string, bool) } diff --git a/helpers.go b/helpers.go index ad27d4a..2eadaba 100644 --- a/helpers.go +++ b/helpers.go @@ -8,42 +8,45 @@ import ( "modernc.org/b/v2" ) -// IncrementIfBuiltinKeyConflict will, if there is a conflict/duplication at the root level (not in a group) with one of -// the built-in keys, add "#01" to the end of the key +// IncrementIfBuiltinKeyConflict is a ResolveKey function that will, if there is +// a conflict/duplication at the root level (not in a group) with one of the +// built-in keys, add "#01" to the end of the key. func IncrementIfBuiltinKeyConflict(groups []string, key string, index int) (string, bool) { - if len(groups) == 0 && DoesBuiltinKeyConflict(key) { - return IncrementKeyName(key, index+1), true // Don't overwrite the built-in attribute keys + if len(groups) == 0 && doesBuiltinKeyConflict(key) { + return incrementKeyName(key, index+1), true // Don't overwrite the built-in attribute keys } - return IncrementKeyName(key, index), true + return incrementKeyName(key, index), true } -// DropIfBuiltinKeyConflict will, if there is a conflict/duplication at the root level (not in a group) with one of the +// DropIfBuiltinKeyConflict is a ResolveKey function that will, if there is a +// conflict/duplication at the root level (not in a group) with one of the // built-in keys, drop the whole attribute func DropIfBuiltinKeyConflict(groups []string, key string, index int) (string, bool) { - if len(groups) == 0 && DoesBuiltinKeyConflict(key) { + if len(groups) == 0 && doesBuiltinKeyConflict(key) { return "", false // Drop the attribute } - return IncrementKeyName(key, index), true + return incrementKeyName(key, index), true } -// KeepIfBuiltinKeyConflict will keep all keys even if there would be a conflict/duplication at the root level (not in a +// KeepIfBuiltinKeyConflict is a ResolveKey function that will keep all keys +// even if there would be a conflict/duplication at the root level (not in a // group) with one of the built-in keys func KeepIfBuiltinKeyConflict(_ []string, key string, index int) (string, bool) { - return IncrementKeyName(key, index), true // Keep all + return incrementKeyName(key, index), true // Keep all } -// DoesBuiltinKeyConflict returns true if the key conflicts with the builtin keys. +// doesBuiltinKeyConflict returns true if the key conflicts with the builtin keys. // This will only be called on all root level (not in a group) attribute keys. -func DoesBuiltinKeyConflict(key string) bool { +func doesBuiltinKeyConflict(key string) bool { if key == slog.TimeKey || key == slog.LevelKey || key == slog.MessageKey || key == slog.SourceKey { return true } return false } -// IncrementKeyName adds a count onto the key name after the first seen. +// incrementKeyName adds a count onto the key name after the first seen. // Example: keyname, keyname#01, keyname#02, keyname#03 -func IncrementKeyName(key string, index int) string { +func incrementKeyName(key string, index int) string { if index == 0 { return key } diff --git a/ignore_handler.go b/ignore_handler.go index 37de6e1..aff6afb 100644 --- a/ignore_handler.go +++ b/ignore_handler.go @@ -13,9 +13,16 @@ type IgnoreHandlerOptions struct { // Comparison function to determine if two keys are equal KeyCompare func(a, b string) int - // Function that will be called on all root level (not in a group) attribute keys. - // Returns the new key value to use, and true to keep the attribute or false to drop it. - // Can be used to drop, keep, or rename any attributes matching the builtin attributes. + // Function that will be called on each attribute and group, to determine + // the key to use. Returns the new key value to use, and true to keep the + // attribute or false to drop it. Can be used to drop, keep, or rename any + // attributes matching the builtin attributes. + // + // The first argument is a list of currently open groups that contain the + // Attr. It must not be retained or modified. + // + // ResolveKey will not be called for the built-in fields on slog.Record + // (ie: time, level, msg, and source). ResolveKey func(groups []string, key string, _ int) (string, bool) } diff --git a/increment_handler.go b/increment_handler.go index de9cc53..fbfe3d4 100644 --- a/increment_handler.go +++ b/increment_handler.go @@ -13,16 +13,22 @@ type IncrementHandlerOptions struct { // Comparison function to determine if two keys are equal KeyCompare func(a, b string) int - // Function that will only be called on all root level (not in a group) attribute keys. - // Returns true if the key conflicts with the builtin keys. - //DoesBuiltinKeyConflict func(key string) bool - - // IncrementKeyName should return a modified key string based on the index (first, second, third instance seen, etc) - //IncrementKeyName func(key string, index int) string - - // Function that will be called on all root level (not in a group) attribute keys. - // Returns the new key value to use, and true to keep the attribute or false to drop it. - // Can be used to drop, keep, or rename any attributes matching the builtin attributes. + // Function that will be called on each attribute and group, to determine + // the key to use. Returns the new key value to use, and true to keep the + // attribute or false to drop it. Can be used to drop, keep, or rename any + // attributes matching the builtin attributes. + // + // For the IncrementHandler, it should return a modified key string based on + // the index (first = 0, second = 1, third = 2, etc). + // If the key is at the root level (groups is empty) and conflicts with a + // builtin key on the slog.Record object (time, level, msg, source), the + // index should be incremented before calculating the modified key string. + // + // The first argument is a list of currently open groups that contain the + // Attr. It must not be retained or modified. + // + // ResolveKey will not be called for the built-in fields on slog.Record + // (ie: time, level, msg, and source). ResolveKey func(groups []string, key string, index int) (string, bool) } diff --git a/overwrite_handler.go b/overwrite_handler.go index e646d5f..1967596 100644 --- a/overwrite_handler.go +++ b/overwrite_handler.go @@ -13,9 +13,16 @@ type OverwriteHandlerOptions struct { // Comparison function to determine if two keys are equal KeyCompare func(a, b string) int - // Function that will be called on all root level (not in a group) attribute keys. - // Returns the new key value to use, and true to keep the attribute or false to drop it. - // Can be used to drop, keep, or rename any attributes matching the builtin attributes. + // Function that will be called on each attribute and group, to determine + // the key to use. Returns the new key value to use, and true to keep the + // attribute or false to drop it. Can be used to drop, keep, or rename any + // attributes matching the builtin attributes. + // + // The first argument is a list of currently open groups that contain the + // Attr. It must not be retained or modified. + // + // ResolveKey will not be called for the built-in fields on slog.Record + // (ie: time, level, msg, and source). ResolveKey func(groups []string, key string, _ int) (string, bool) } diff --git a/resolve_keys_replace_attrs.go b/resolve_keys_replace_attrs.go index 56bdf17..173e0ae 100644 --- a/resolve_keys_replace_attrs.go +++ b/resolve_keys_replace_attrs.go @@ -7,7 +7,7 @@ import ( ) // JoinResolveKey can be used to join together many slogdedup middlewares -// ...HandlerOptions.ResolveKey functions into a single one that applies all the +// xHandlerOptions.ResolveKey functions into a single one that applies all the // rules in order. func JoinResolveKey(resolveKeyFunctions ...func(groups []string, key string, index int) (string, bool)) func(groups []string, key string, index int) (string, bool) { if len(resolveKeyFunctions) == 0 { @@ -26,7 +26,7 @@ func JoinResolveKey(resolveKeyFunctions ...func(groups []string, key string, ind if key != originalKey { return key, ok } - return IncrementKeyName(key, index), ok + return incrementKeyName(key, index), ok } } @@ -75,7 +75,6 @@ var sinkGraylog = sink{ }, } - // ResolveKeyStackdriver returns a ResolveKey function works for Stackdriver // (aka Google Cloud Operations, aka GCP Log Explorer). func ResolveKeyStackdriver() func(groups []string, key string, index int) (string, bool) { @@ -171,7 +170,7 @@ type attrReplacer struct { } // resolveKeys returns a closure that can be used with any slogdedup middlewares -// ...HandlerOptions.ResolveKey. Its purpose is to replace the key on any +// xHandlerOptions.ResolveKey. Its purpose is to replace the key on any // attributes or groups, except for the builtin attributes. Using replaceAttr on // the final handler/sink is still required, in order to replace the builtin // attribute keys. @@ -202,7 +201,7 @@ func resolveKeys(dest sink) func(groups []string, key string, index int) (string // Check builtins last for _, builtin := range dest.builtins { if key == builtin { - return IncrementKeyName(key, index+1), true + return incrementKeyName(key, index+1), true } } return key, true From e470dd2dff34ae8a263bbce5a811833cf094d7e0 Mon Sep 17 00:00:00 2001 From: Chris Duncan Date: Thu, 21 Mar 2024 10:11:16 -0600 Subject: [PATCH 11/11] Update docs and Readme and examples --- README.md | 138 ++++++++++++++++++++++++++--- doc.go | 34 ++++++- resolve_keys_replace_attrs.go | 20 ++--- resolve_keys_replace_attrs_test.go | 4 +- 4 files changed, 167 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 7641f9f..5f8e415 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,22 @@ Outputs: "duplicated": "two" } ``` -With this in mind, this repo was created with several different ways of deduplicating the keys. +With this in mind, this repo was created with several different ways of deduplicating the keys: overwriting, ignoring, appending, incrementing. +For example, incrementing: +```json +{ + "time": "2024-03-21T09:33:25Z", + "level": "INFO", + "msg": "this is the dedup incrementer handler", + "duplicated": "zero", + "duplicated#01": "one", + "duplicated#02": "two" +} +``` + +Additionally, this library includes convenience methods for formatting output to +match what is expected for various log aggregation tools (such as Graylog), as +well as cloud providers (such as Stackdriver / Google Cloud Operations / GCP Log Explorer). ### Other Great SLOG Utilities - [slogctx](https://github.com/veqryn/slog-context): Add attributes to context and have them automatically added to all log lines. Work with a logger stored in context. @@ -66,7 +81,7 @@ logger.Info("this is the dedup overwrite handler", Outputs: ```json { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup overwrite handler", "duplicated": "two" @@ -85,7 +100,7 @@ logger.Info("this is the dedup ignore handler", Outputs: ```json { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup ignore handler", "duplicated": "zero" @@ -104,7 +119,7 @@ logger.Info("this is the dedup incrementer handler", Outputs: ```json { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup incrementer handler", "duplicated": "zero", @@ -125,7 +140,7 @@ logger.Info("this is the dedup appender handler", Outputs: ```json { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup appender handler", "duplicated": [ @@ -136,7 +151,35 @@ Outputs: } ``` +### Using ResolveKey and ReplaceAttr +#### Stackdriver (GCP Cloud Operations / Google Log Explorer) +```go +logger := slog.New(slogdedup.NewOverwriteHandler( + slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + AddSource: true, + ReplaceAttr: slogdedup.ReplaceAttrStackdriver(), // Needed for builtin's + }), + &slogdedup.OverwriteHandlerOptions{ResolveKey: slogdedup.ResolveKeyStackdriver()}, // Needed for everything else, and deduplication +)) +logger.Warn("this is the main message", slog.String("duplicated", "zero"), slog.String("duplicated", "one")) +``` +Outputs: +```json +{ + "time": "2024-03-21T09:59:19.652284-06:00", + "severity": "WARNING", + "logging.googleapis.com/sourceLocation": { + "function": "main.main", + "file": "/go/src/github.com/veqryn/slog-dedup/cmd/replacers/cmd.go", + "line": "19" + }, + "message": "this is the main message", + "duplicated": "one" +} +``` + ## Full Example Main File +### Basic Use of Each Handler ```go package main @@ -154,7 +197,7 @@ func main() { /* { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup overwrite handler", "duplicated": "two" @@ -172,7 +215,7 @@ func main() { /* { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup ignore handler", "duplicated": "zero" @@ -190,7 +233,7 @@ func main() { /* { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup incrementer handler", "duplicated": "zero", @@ -210,7 +253,7 @@ func main() { /* { - "time": "2023-10-03T01:30:00Z", + "time": "2024-03-21T09:33:25Z", "level": "INFO", "msg": "this is the dedup appender handler", "duplicated": [ @@ -228,8 +271,79 @@ func main() { } ``` +### Using ResolveKey and ReplaceAttr +```go +package main + +import ( + "log/slog" + "os" + + slogdedup "github.com/veqryn/slog-dedup" +) + +func main() { + // Example sending logs of Stackdriver (GCP Cloud Operations / Google Log Explorer) + // which then pipes the data to Graylog. + + // First, create a function to resolve/replace attribute keys before deduplication, + // which will also ensure the builtin keys are unused by non-builtin attributes: + resolveKey := slogdedup.JoinResolveKey( + slogdedup.ResolveKeyStackdriver(), + slogdedup.ResolveKeyGraylog(), + ) + + // Second, create a function to replace the builtin record attributes + // (time, level, msg, source) with the appropriate keys and values for + // Stackdriver and Graylog: + replaceAttr := slogdedup.JoinReplaceAttr( + slogdedup.ReplaceAttrStackdriver(), + slogdedup.ReplaceAttrGraylog(), + ) + + // Next create the final handler (the sink), which is a json handler, + // using the replaceAttr function we just made: + jsonHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + AddSource: true, + Level: slog.LevelDebug, + ReplaceAttr: replaceAttr, + }) + + // Now create any one of the dedup middleware handlers, using the + // resolveKey function we just made, then set it as the default logger: + dedupHandler := slogdedup.NewOverwriteHandler( + jsonHandler, + &slogdedup.OverwriteHandlerOptions{ + KeyCompare: slogdedup.CaseSensitiveCmp, // this is the default + ResolveKey: resolveKey, // use the key resolver for stackdriver/graylog + }, + ) + slog.SetDefault(slog.New(dedupHandler)) + + /* + { + "time": "2024-03-21T09:53:43Z", + "severity": "WARNING", + "logging.googleapis.com/sourceLocation": { + "function": "main.main", + "file": "/go/src/github.com/veqryn/slog-dedup/cmd/replacers/cmd.go", + "line": "56" + }, + "message": "this is the main message", + "duplicated": "one" + } + */ + slog.Warn("this is the main message", slog.String("duplicated", "zero"), slog.String("duplicated", "one")) +} +``` + ## Breaking Changes -### O.1.0 -> 0.2.0 +### O.3.x -> 0.4.0 +`ResolveBuiltinKeyConflict`,`DoesBuiltinKeyConflict`, and `IncrementKeyName` have +all been unified into a single function: `ResolveKey`. + + +### O.1.x -> 0.2.0 Package renamed from `dedup` to `slogdedup`. To fix, change this: ```go @@ -249,7 +363,7 @@ This library has convenience methods that allow it to interoperate with [github. in order to easily setup slog workflows such as pipelines, fanout, routing, failover, etc. ```go slog.SetDefault(slog.New(slogmulti. - Pipe(slogcontext.NewMiddleware(&slogcontext.HandlerOptions{})). + Pipe(slogctx.NewMiddleware(&slogctx.HandlerOptions{})). Pipe(slogdedup.NewOverwriteMiddleware(&slogdedup.OverwriteHandlerOptions{})). Handler(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{})), )) @@ -262,4 +376,4 @@ Using an overwrite handler allows a slightly different style of logging that is These handlers will correctly deal with sub-loggers, whether using `WithAttrs()` or `WithGroup()`. It will even handle groups injected as attributes using `slog.Group()`. Due to the lack of a `slog.Slice` type/kind, the `AppendHandler` has a special case where groups that are inside of slices/arrays are turned into a `map[string]any{}` slog attribute before being passed to the final handler. ### The Built-In Fields (time, level, msg, source) -Because this handler is a middleware, it must pass a `slog.Record` to the final handler. The built-in attributes for time, level, msg, and source are treated separately, and have their own fields on the `slog.Record` struct. It would therefore be impossible to deduplicate these, if we didn't handle these as a special case. The increment handler considers that these four keys are always taken at the root level, and any attributes using those keys will start with the #01 increment on their key name. The other handlers can be customized using their options struct to either increment the name (default), overwrite, or allow the duplicates for the builtin keys. You can also customize this behavior by passing your own functions to the options struct (same for log handlers that use different keys for the built-in fields). +Because this handler is a middleware, it must pass a `slog.Record` to the final handler. The built-in attributes for time, level, msg, and source are treated separately, and have their own fields on the `slog.Record` struct. It would therefore be impossible to deduplicate these, if we didn't handle these as a special case. The increment handler considers that these four keys are always taken at the root level, and any attributes using those keys will start with the #01 increment on their key name. The other handlers can be customized using their options struct to either increment the name (default), drop old attributes using those keys (overwrite with the final slog.Record builtins), or allow the duplicates for the builtin keys. You can also customize this behavior by passing your own functions to the options struct (same for log handlers that use different keys for the built-in fields). diff --git a/doc.go b/doc.go index 1345ae1..8d80f4f 100644 --- a/doc.go +++ b/doc.go @@ -5,6 +5,10 @@ The main impetus behind this package is because most JSON tools do not like dupl properties/fields. Some of them will give errors or fail to parse the log line, and some may even crash. Unfortunately the default behavior of the stdlib slog handlers is to allow duplicate keys. +Additionally, this library includes convenience methods for formatting output to +match what is expected for various log aggregation tools (such as Graylog), as +well as cloud providers (such as Stackdriver / Google Cloud Operations / GCP Log Explorer). + Usage: // OverwriteHandler @@ -12,7 +16,7 @@ Usage: slog.SetDefault(slog.New(overwriter)) // { - // "time": "2023-10-03T01:30:00Z", + // "time": "2024-03-21T09:33:25Z", // "level": "INFO", // "msg": "this is the dedup overwrite handler", // "duplicated": "two" @@ -28,7 +32,7 @@ Usage: slog.SetDefault(slog.New(ignorer)) // { - // "time": "2023-10-03T01:30:00Z", + // "time": "2024-03-21T09:33:25Z", // "level": "INFO", // "msg": "this is the dedup ignore handler", // "duplicated": "zero" @@ -44,7 +48,7 @@ Usage: slog.SetDefault(slog.New(incrementer)) // { - // "time": "2023-10-03T01:30:00Z", + // "time": "2024-03-21T09:33:25Z", // "level": "INFO", // "msg": "this is the dedup incrementer handler", // "duplicated": "zero", @@ -62,7 +66,7 @@ Usage: slog.SetDefault(slog.New(appender)) // { - // "time": "2023-10-03T01:30:00Z", + // "time": "2024-03-21T09:33:25Z", // "level": "INFO", // "msg": "this is the dedup appender handler", // "duplicated": [ @@ -76,5 +80,27 @@ Usage: slog.String("duplicated", "one"), slog.String("duplicated", "two"), ) + + + logger := slog.New(slogdedup.NewOverwriteHandler( + slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + AddSource: true, + ReplaceAttr: slogdedup.ReplaceAttrStackdriver(), // Needed for builtin's + }), + &slogdedup.OverwriteHandlerOptions{ResolveKey: slogdedup.ResolveKeyStackdriver()}, // Needed for everything else, and deduplication + )) + + // { + // "time": "2024-03-21T09:59:19.652284-06:00", + // "severity": "WARNING", + // "logging.googleapis.com/sourceLocation": { + // "function": "main.main", + // "file": "/go/src/github.com/veqryn/slog-dedup/cmd/replacers/cmd.go", + // "line": "19" + // }, + // "message": "this is the main message", + // "duplicated": "one" + // } + logger.Warn("this is the main message", slog.String("duplicated", "zero"), slog.String("duplicated", "one")) */ package slogdedup diff --git a/resolve_keys_replace_attrs.go b/resolve_keys_replace_attrs.go index 173e0ae..6d48520 100644 --- a/resolve_keys_replace_attrs.go +++ b/resolve_keys_replace_attrs.go @@ -1,7 +1,6 @@ package slogdedup import ( - "fmt" "log/slog" "strconv" ) @@ -102,19 +101,19 @@ var sinkStackdriver = sink{ switch lvl := v.Any().(type) { case slog.Level: if lvl <= slog.LevelDebug { - return slog.StringValue("DEBUG") + return slog.StringValue("DEBUG") // -4 } else if lvl <= slog.LevelInfo { - return slog.StringValue("INFO") - } else if lvl < slog.LevelWarn { - return slog.StringValue("NOTICE") - } else if lvl == slog.LevelWarn { - return slog.StringValue("WARNING") + return slog.StringValue("INFO") // 0 + } else if lvl <= slog.LevelInfo+2 { + return slog.StringValue("NOTICE") // 2 + } else if lvl <= slog.LevelWarn { + return slog.StringValue("WARNING") // 4 } else if lvl <= slog.LevelError { - return slog.StringValue("ERROR") + return slog.StringValue("ERROR") // 8 } else if lvl <= slog.LevelError+4 { - return slog.StringValue("CRITICAL") + return slog.StringValue("CRITICAL") // 12 } else if lvl <= slog.LevelError+8 { - return slog.StringValue("ALERT") + return slog.StringValue("ALERT") // 16 } return slog.StringValue("EMERGENCY") default: @@ -146,7 +145,6 @@ var sinkStackdriver = sink{ Line: strconv.Itoa(source.Line), }) default: - fmt.Printf("SOURCE: %T: %#+v\n", source, source) return v } }}, diff --git a/resolve_keys_replace_attrs_test.go b/resolve_keys_replace_attrs_test.go index 870b58e..2015617 100644 --- a/resolve_keys_replace_attrs_test.go +++ b/resolve_keys_replace_attrs_test.go @@ -7,7 +7,7 @@ import ( "testing" ) -func TestFoo(t *testing.T) { +func TestResolveKeyReplaceAttr(t *testing.T) { t.Parallel() resolvers := JoinResolveKey( @@ -40,7 +40,7 @@ func TestFoo(t *testing.T) { }, { hander: NewIgnoreHandler(tester, &IgnoreHandlerOptions{ResolveKey: resolvers}), - expected: `{"time":"2023-09-29T13:00:59Z","severity":"WARNING","logging.googleapis.com/sourceLocation":{"function":"github.com/veqryn/slog-dedup.logComplex","file":"github.com/veqryn/slog-dedup/helpers_test.go","line":"85"},"message":"main message","arg1":"with1arg1","arg2":"with1arg2","arg3":"with1arg3","arg4":"with2arg4","group1":"with2group1","logging.googleapis.com/sourceLocation#01":"with1source","message#01":"with2msg","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","severity#01":"with2level","sourceLoc#01":"sourceLocArg","time#01":"with1time","timestampRenamed":"timestampArg","typed":"overwritten","with1":"arg0","with2":"arg0"`, + expected: `{"time":"2023-09-29T13:00:59Z","severity":"WARNING","logging.googleapis.com/sourceLocation":{"function":"github.com/veqryn/slog-dedup.logComplex","file":"github.com/veqryn/slog-dedup/helpers_test.go","line":"85"},"message":"main message","arg1":"with1arg1","arg2":"with1arg2","arg3":"with1arg3","arg4":"with2arg4","group1":"with2group1","logging.googleapis.com/sourceLocation#01":"with1source","message#01":"with2msg","msg#01":"prexisting01","msg#01a":"seekbug01a","msg#02":"seekbug02","severity#01":"with2level","sourceLoc#01":"sourceLocArg","time#01":"with1time","timestampRenamed":"timestampArg","typed":"overwritten","with1":"arg0","with2":"arg0"}`, }, { hander: NewAppendHandler(tester, &AppendHandlerOptions{ResolveKey: resolvers}),