From be33749b8634d691ca06b57a04b3d4b86712ed09 Mon Sep 17 00:00:00 2001 From: Matthew Schmidt Date: Mon, 20 May 2024 14:34:45 -0400 Subject: [PATCH 1/2] Support UnmarshalCSVWithFields in readEach - Slight rearrangment compared to readTo to place it above the fieldInfo check; if a type is going to completely handle unmarshalling by implementing this interface, we shouldn't care that it has matching csv tagged fields or not - Also add missing default value setting in readEach --- decode.go | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/decode.go b/decode.go index 24d49d0..049a467 100644 --- a/decode.go +++ b/decode.go @@ -289,8 +289,13 @@ func readEach(decoder SimpleDecoder, errHandler ErrorHandler, c interface{}) err return err } } + + var withFieldsOK bool + var fieldTypeUnmarshallerWithKeys TypeUnmarshalCSVWithFields + i := 0 for { + objectIface := reflect.New(outValue.Type().Elem()).Interface() line, err := decoder.GetCSVRow() if err == io.EOF { break @@ -299,8 +304,31 @@ func readEach(decoder SimpleDecoder, errHandler ErrorHandler, c interface{}) err } outInner := createNewOutInner(outInnerWasPointer, outInnerType) for j, csvColumnContent := range line { + + if outInner.CanInterface() { + fieldTypeUnmarshallerWithKeys, withFieldsOK = objectIface.(TypeUnmarshalCSVWithFields) + if withFieldsOK { + if err := fieldTypeUnmarshallerWithKeys.UnmarshalCSVWithFields(headers[j], csvColumnContent); err != nil { + parseError := csv.ParseError{ + Line: i + 2, //add 2 to account for the header & 0-indexing of arrays + Column: j + 1, + Err: err, + } + return &parseError + } + + continue + } + } + if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name - if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct + + value := csvColumnContent + if value == "" { + value = fieldInfo.defaultValue + } + + if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, value, fieldInfo.omitEmpty); err != nil { // Set field of struct parseError := &csv.ParseError{ Line: i + 2, //add 2 to account for the header & 0-indexing of arrays Column: j + 1, @@ -313,6 +341,12 @@ func readEach(decoder SimpleDecoder, errHandler ErrorHandler, c interface{}) err } } } + + if withFieldsOK { + reflectedObject := reflect.ValueOf(objectIface) + outInner = reflectedObject.Elem() + } + outValue.Send(outInner) i++ } From 8538a3010ebd64320a1c2c9c6cff3d2f0f1a879d Mon Sep 17 00:00:00 2001 From: Matthew Schmidt Date: Mon, 20 May 2024 15:48:46 -0400 Subject: [PATCH 2/2] Add tests, allow TypeUnmarshalCSVWithFields to work in readTo without matching csv tags --- decode.go | 26 +++++----- decode_test.go | 109 ++++++++++++++++++++++++++++++++++++++++- sample_structs_test.go | 6 +++ 3 files changed, 127 insertions(+), 14 deletions(-) diff --git a/decode.go b/decode.go index 049a467..e91c595 100644 --- a/decode.go +++ b/decode.go @@ -203,22 +203,22 @@ func readToWithErrorHandler(decoder Decoder, errHandler ErrorHandler, out interf objectIface := reflect.New(outValue.Index(i).Type()).Interface() outInner := createNewOutInner(outInnerWasPointer, outInnerType) for j, csvColumnContent := range csvRow { - if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name - - if outInner.CanInterface() { - fieldTypeUnmarshallerWithKeys, withFieldsOK = objectIface.(TypeUnmarshalCSVWithFields) - if withFieldsOK { - if err := fieldTypeUnmarshallerWithKeys.UnmarshalCSVWithFields(fieldInfo.getFirstKey(), csvColumnContent); err != nil { - parseError := csv.ParseError{ - Line: i + 2, //add 2 to account for the header & 0-indexing of arrays - Column: j + 1, - Err: err, - } - return &parseError + if outInner.CanInterface() { + fieldTypeUnmarshallerWithKeys, withFieldsOK = objectIface.(TypeUnmarshalCSVWithFields) + if withFieldsOK { + if err := fieldTypeUnmarshallerWithKeys.UnmarshalCSVWithFields(headers[j], csvColumnContent); err != nil { + parseError := csv.ParseError{ + Line: i + 2, //add 2 to account for the header & 0-indexing of arrays + Column: j + 1, + Err: err, } - continue + return &parseError } + continue } + } + + if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name value := csvColumnContent if value == "" { value = fieldInfo.defaultValue diff --git a/decode_test.go b/decode_test.go index 7eebc73..d010c31 100644 --- a/decode_test.go +++ b/decode_test.go @@ -467,6 +467,57 @@ L: } } +// Test_readEach_TypeUnmarshalCSVWithFields test that readEach works with type implementing TypeUnmarshalCSVWithFields for unmarshalling +func Test_readEach_TypeUnmarshalCSVWithFields(t *testing.T) { + b := bytes.NewBufferString(`foo,bar,baz,frop +bb,1,cc,3.14 +gg,2,hh,4`) + d := newSimpleDecoderFromReader(b) + + c := make(chan UnmarshalCSVWithFieldsSample) + e := make(chan error) + var samples []UnmarshalCSVWithFieldsSample + go func() { + if err := readEach(d, nil, c); err != nil { + e <- err + } + }() +L: + for { + select { + case err := <-e: + t.Fatal(err) + case v, ok := <-c: + if !ok { + break L + } + samples = append(samples, v) + + } + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + expected := UnmarshalCSVWithFieldsSample{ + Foo: "bb", + Bar: 1, + Baz: "cc", + Frop: 314, + } + if expected != samples[0] { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = UnmarshalCSVWithFieldsSample{ + Foo: "gg", + Bar: 2, + Baz: "hh", + Frop: 400, + } + if expected != samples[1] { + t.Fatalf("expected first sample %v, got %v", expected, samples[1]) + } +} + func Test_maybeMissingStructFields(t *testing.T) { structTags := []fieldInfo{ {keys: []string{"foo"}}, @@ -712,7 +763,7 @@ func (rf *RenamedFloat64Unmarshaler) UnmarshalCSV(csv string) (err error) { return nil } -// TestUnmarshalCSVWithFields test that the TestUnmarshalCSVWithFields interface to marshall all the fields works +// TestUnmarshalCSVWithFields test that the TestUnmarshalCSVWithFields interface to unmarshal all the fields works func TestUnmarshalCSVWithFields(t *testing.T) { b := []byte(`foo,bar,baz,frop bar,1,zip,3.14 @@ -800,6 +851,62 @@ func (e UnmarshalError) Error() string { return e.msg } +// TestUnmarshalCSVWithFieldsNoTags test that the TypeUnmarshalCSVWithFields interface works for types without matching csv tags (or no csv tags at all) +func TestUnmarshalCSVWithFieldsNoTags(t *testing.T) { + b := []byte(`foo,bar,baz,frop +bar,1,zip,3.14 +baz,2,zap,4.00`) + var samples []NoTagsSample + err := UnmarshalBytes(b, &samples) + if err != nil { + t.Fatalf("UnmarshalCSVWithFields() -> UnmarshalBytes() %v", err) + } + + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + + if len(samples[0].Fields) != 4 { + t.Fatalf("expected 4 sample fields in map, got %d", len(samples[0].Fields)) + } + + checkFieldMap := func(expected map[string]string, actual map[string]string) { + for expectedKey, expectedValue := range expected { + if sampleValue, ok := actual[expectedKey]; ok { + if sampleValue != expectedValue { + t.Fatalf("expected %s key in map to have %s value, got %s", expectedKey, expectedValue, sampleValue) + } + } else { + t.Fatalf("expected map to have key %s", expectedKey) + } + } + } + + expectedFields := map[string]string{ + "foo": "bar", + "bar": "1", + "baz": "zip", + "frop": "3.14", + } + checkFieldMap(expectedFields, samples[0].Fields) + + expectedFields = map[string]string{ + "foo": "baz", + "bar": "2", + "baz": "zap", + "frop": "4.00", + } + checkFieldMap(expectedFields, samples[1].Fields) +} + +func (u *NoTagsSample) UnmarshalCSVWithFields(key, value string) error { + if u.Fields == nil { + u.Fields = map[string]string{} + } + u.Fields[key] = value + return nil +} + func TestMultipleStructTags(t *testing.T) { b := bytes.NewBufferString(`foo,BAR,Baz e,3,b`) diff --git a/sample_structs_test.go b/sample_structs_test.go index ca95cff..b9c509b 100644 --- a/sample_structs_test.go +++ b/sample_structs_test.go @@ -188,3 +188,9 @@ type NestedSample struct { type NestedEmbedSample struct { InnerStruct } + +type NoTagsSample struct { + Fields map[string]string +} + +var _ TypeUnmarshalCSVWithFields = (*NoTagsSample)(nil)