Skip to content

Commit

Permalink
CSV Writer: Allow different types in output
Browse files Browse the repository at this point in the history
Adjust the csvio.Writer so that it can handle records with the same
field names but different types.

Closes #4781
  • Loading branch information
mattnibs committed Nov 17, 2023
1 parent bedcc95 commit 2923261
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 2 deletions.
17 changes: 15 additions & 2 deletions zio/csvio/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"slices"
"strings"

"github.com/brimdata/zed"
Expand All @@ -19,6 +20,7 @@ type Writer struct {
writer io.WriteCloser
encoder *csv.Writer
flattener *expr.Flattener
types map[int]struct{}
first *zed.TypeRecord
strings []string
}
Expand All @@ -32,6 +34,7 @@ func NewWriter(w io.WriteCloser) *Writer {
writer: w,
encoder: csv.NewWriter(w),
flattener: expr.NewFlattener(zed.NewContext()),
types: make(map[int]struct{}),
}
}

Expand Down Expand Up @@ -62,8 +65,12 @@ func (w *Writer) Write(rec *zed.Value) error {
if err := w.encoder.Write(hdr); err != nil {
return err
}
} else if rec.Type != w.first {
return ErrNotDataFrame
}
if _, ok := w.types[rec.Type.ID()]; !ok {
if !fieldsEqual(w.first.Fields, rec.Fields()) {
return ErrNotDataFrame
}
w.types[rec.Type.ID()] = struct{}{}
}
w.strings = w.strings[:0]
fields := rec.Fields()
Expand Down Expand Up @@ -96,3 +103,9 @@ func formatValue(typ zed.Type, bytes zcode.Bytes) string {
}
return zson.FormatValue(zed.NewValue(typ, bytes))
}

func fieldsEqual(a, b []zed.Field) bool {
return slices.EqualFunc(a, b, func(a, b zed.Field) bool {
return a.Name == b.Name
})
}
14 changes: 14 additions & 0 deletions zio/csvio/ztests/different-types.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
zed: '*'

input: |
{a:0}
{a:"foo"}
{a:127.0.0.1}
output-flags: -f csv

output: |
a
0
foo
127.0.0.1

0 comments on commit 2923261

Please sign in to comment.