From 71dfc7000a6fbd1ab43f38c0b4764763c06db65e Mon Sep 17 00:00:00 2001 From: Alec Sammon Date: Thu, 30 May 2024 12:12:14 +0100 Subject: [PATCH 1/2] Allow using a custom KV store to keep consistent JSON ordering --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ kvstore.go | 17 +++++++++++++++++ sheriff.go | 38 ++++++++++++++++++++++++++++++-------- sheriff_test.go | 12 ++++++------ 4 files changed, 95 insertions(+), 14 deletions(-) create mode 100644 kvstore.go diff --git a/README.md b/README.md index 23820d5..7aabfd0 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,48 @@ func main() { // ] ``` +## Output ordering + +Sheriff converts the input struct into a basic structure using `map[string]interface{}`. This means that the generated +JSON will not have the same ordering as the input struct. If you need to have a specific ordering then a custom +implementation of the `KVStoreFactory` can be passed as an option. +```go +package main + +import ( + "github.com/liip/sheriff/v2" + orderedmap "github.com/wk8/go-ordered-map/v2" +) + +type OrderedMap struct { + *orderedmap.OrderedMap[string, interface{}] +} + +func NewOrderedMap() *OrderedMap { + return &OrderedMap{orderedmap.New[string, interface{}]()} +} + +func (om *OrderedMap) Set(k string, v interface{}) { + om.OrderedMap.Set(k, v) +} + +func (om *OrderedMap) Each(f func(k string, v interface{})) { + for pair := om.Newest(); pair != nil; pair = pair.Prev() { + f(pair.Key, pair.Value) + } +} + +func main() { + opt := &sheriff.Options{ + KVStoreFactory: func() sheriff.KVStore { + return NewOrderedMap() + }, + } + + // ... +} +``` + ## Benchmarks There's a simple benchmark in `bench_test.go` which compares running sheriff -> JSON versus just marshalling into JSON diff --git a/kvstore.go b/kvstore.go new file mode 100644 index 0000000..7f5e319 --- /dev/null +++ b/kvstore.go @@ -0,0 +1,17 @@ +package sheriff + +// kvStore is the default implementation of the KVStore interface that sheriff converts a struct into. +// It is the fastest option, but does result in a re-ordering of the final JSON properties. +type kvStore map[string]interface{} + +// Set inserts the value into the map at the given key. +func (m kvStore) Set(k string, v interface{}) { + m[k] = v +} + +// Each applies the callback function to each element in the map. +func (m kvStore) Each(f func(k string, v interface{})) { + for k, v := range m { + f(k, v) + } +} diff --git a/sheriff.go b/sheriff.go index c2d19a5..c6d1995 100644 --- a/sheriff.go +++ b/sheriff.go @@ -10,6 +10,15 @@ import ( "github.com/hashicorp/go-version" ) +// A KVStore is a simple key-value store. +// The default implementation uses a map[string]interface{}. +// This is fast, however will lead to inconsistent ordering of the keys in the generated JSON. +// A custom implementation can be used to maintain the order of the keys. +type KVStore interface { + Set(k string, v interface{}) + Each(f func(k string, v interface{})) +} + // A FieldFilter is a function that decides whether a field should be marshalled or not. // If it returns true, the field will be marshalled, otherwise it will be skipped. type FieldFilter func(field reflect.StructField) (bool, error) @@ -38,6 +47,12 @@ type Options struct { // This option is false by default. IncludeEmptyTag bool + // The KVStoreFactory is a function that returns a new KVStore. + // The default implementation uses a map[string]interface{}, which is fast but does not maintain the order of the + // keys. + // A custom implementation can be used to maintain the order of the keys, i.e. using github.com/wk8/go-ordered-map + KVStoreFactory func() KVStore + // This is used internally so that we can propagate anonymous fields groups tag to all child field. nestedGroupsMap map[string][]string } @@ -81,6 +96,12 @@ func Marshal(options *Options, data interface{}) (interface{}, error) { options.FieldFilter = createDefaultFieldFilter(options) } + if options.KVStoreFactory == nil { + options.KVStoreFactory = func() KVStore { + return kvStore{} + } + } + if t.Kind() == reflect.Ptr { // follow pointer t = t.Elem() @@ -94,7 +115,7 @@ func Marshal(options *Options, data interface{}) (interface{}, error) { return marshalValue(options, v) } - dest := make(map[string]interface{}) + dest := options.KVStoreFactory() for i := 0; i < t.NumField(); i++ { field := t.Field(i) @@ -172,13 +193,13 @@ func Marshal(options *Options, data interface{}) (interface{}, error) { // when a composition field we want to bring the child // nodes to the top - nestedVal, ok := v.(map[string]interface{}) + nestedVal, ok := v.(KVStore) if isEmbeddedField && ok { - for key, value := range nestedVal { - dest[key] = value - } + nestedVal.Each(func(k string, v interface{}) { + dest.Set(k, v) + }) } else { - dest[jsonTag] = v + dest.Set(jsonTag, v) } } @@ -303,13 +324,14 @@ func marshalValue(options *Options, v reflect.Value) (interface{}, error) { if mapKeys[0].Kind() != reflect.String { return nil, MarshalInvalidTypeError{t: mapKeys[0].Kind(), data: val} } - dest := make(map[string]interface{}) + + dest := options.KVStoreFactory() for _, key := range mapKeys { d, err := marshalValue(options, v.MapIndex(key)) if err != nil { return nil, err } - dest[key.String()] = d + dest.Set(key.String(), d) } return dest, nil } diff --git a/sheriff_test.go b/sheriff_test.go index bddbecf..72b5a62 100644 --- a/sheriff_test.go +++ b/sheriff_test.go @@ -846,13 +846,12 @@ func TestMarshal_User(t *testing.T) { TestS: "test", } - v, err := Marshal(&Options{}, j) + m, err := Marshal(&Options{}, j) assert.NoError(t, err) - assert.Equal(t, map[string]interface{}{"test": "12", "testb": "true", "testf": "12", "tests": "test"}, v) - d, err := json.Marshal(j) + d, err := json.Marshal(m) assert.NoError(t, err) - assert.Equal(t, `{"test":"12","testb":"true","testf":"12","tests":"\"test\""}`, string(d)) + assert.Equal(t, `{"test":"12","testb":"true","testf":"12","tests":"test"}`, string(d)) } func TestMarshal_CustomFieldFilter(t *testing.T) { @@ -873,6 +872,7 @@ func TestMarshal_CustomFieldFilter(t *testing.T) { m, err := Marshal(o, v) assert.NoError(t, err) - // ensure the "secret" value is not present in the marshalled map - assert.Equal(t, map[string]interface{}{"test": "teststring"}, m) + d, err := json.Marshal(m) + assert.NoError(t, err) + assert.Equal(t, `{"test":"teststring"}`, string(d)) } From 3ffae7adf708abe312aaad4baaf3c1b655de55c5 Mon Sep 17 00:00:00 2001 From: Alec Sammon Date: Fri, 31 May 2024 09:59:52 +0100 Subject: [PATCH 2/2] * tweak Readme with AI suggestions! --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 7aabfd0..34a50bc 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,11 @@ func main() { Sheriff converts the input struct into a basic structure using `map[string]interface{}`. This means that the generated JSON will not have the same ordering as the input struct. If you need to have a specific ordering then a custom implementation of the `KVStoreFactory` can be passed as an option. + +Providing a custom KV Store is likely to have a negative impact on performance, as such it should be used only when +necessary. + +For example: ```go package main