refactoring column types; formatter tests

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
  - name: Check out code
  uses: actions/checkout@master
  - name: Release
- uses: gavincabbage/ghaction-goreleaser@signing
+ uses: goreleaser/goreleaser-action@master
  with:
  args: release
  key: ${{ secrets.RELEASE_PRIVATE_KEY }}

chiv.go

@@ -55,7 +55,7 @@ func ArchiveRowsWithContext(ctx context.Context, rows Rows, s3 Uploader, bucket
 type Archiver struct {
  db Database
  s3 Uploader
- format FormatterFunc
+ format Format
  key string
  extension string
  null []byte
@@ -140,7 +140,7 @@ func (a *Archiver) download(ctx context.Context, rows Rows, w io.WriteCloser) (e
  }
  }()
 
- columns, err := rows.ColumnTypes()
+ columns, err := interfaced(rows.ColumnTypes())
  if err != nil {
  return err
  }
@@ -234,3 +234,12 @@ func (a *Archiver) upload(ctx context.Context, r io.ReadCloser, table string, bu
 
  return err
 }
+
+func interfaced(in []*sql.ColumnType, err error) ([]Column, error) {
+ out := make([]Column, len(in))
+ for i := range in {
+ out[i] = in[i]
+ }
+
+ return out, err
+}

chiv_benchmark_test.go

@@ -19,12 +19,12 @@ func BenchmarkArchiver_Archive(b *testing.B) {
  benchmarks = []int{1, 10, 100, 1_000, 5_000, 10_000, 1_000_000}
  ctx = context.Background()
  rows = &benchmarkRows{
- columnTypes: make([]*sql.ColumnType, 10),
+ columnTypes: make([]chiv.Column, 10),
  column: sql.RawBytes("column_value"),
  }
  uploader = &uploader{}
  bucket = "benchmark_bucket"
- format = chiv.WithFormat(formatterFunc(&benchmarkFormatter{}, nil))
+ format = chiv.WithFormat(format(&benchmarkFormatter{}, nil))
  )
 
  for _, count := range benchmarks {
@@ -41,12 +41,12 @@ func BenchmarkArchiver_Archive(b *testing.B) {
 }
 
 type benchmarkRows struct {
- columnTypes []*sql.ColumnType
+ columnTypes []chiv.Column
  column sql.RawBytes
  ndx, max int
 }
 
-func (r *benchmarkRows) ColumnTypes() ([]*sql.ColumnType, error) {
+func (r *benchmarkRows) ColumnTypes() ([]chiv.Column, error) {
  return r.columnTypes, nil
 }
 

chiv_integration_test.go

@@ -381,6 +381,8 @@ const (
 )
 
 func newDB(t testing.TB, driver string, url string) *sql.DB {
+ t.Helper()
+
  db, err := sql.Open(driver, url)
  if err != nil {
  t.Fatal(err)
@@ -401,6 +403,8 @@ func newDB(t testing.TB, driver string, url string) *sql.DB {
 }
 
 func newS3Client(t testing.TB, region string, endpoint string) *s3.S3 {
+ t.Helper()
+
  awsConfig := aws.NewConfig().
  WithRegion(region).
  WithDisableSSL(true).
@@ -427,6 +431,8 @@ func exec(t testing.TB, db *sql.DB, statements string) {
 }
 
 func createBucket(t testing.TB, client *s3.S3, name string) {
+ t.Helper()
+
  if _, err := client.CreateBucket(&s3.CreateBucketInput{
  Bucket: aws.String(name),
  }); err != nil {
@@ -435,6 +441,8 @@ func createBucket(t testing.TB, client *s3.S3, name string) {
 }
 
 func deleteBucket(t testing.TB, client *s3.S3, name string) {
+ t.Helper()
+
  // we could do this more cleanly with BatchDeleteIterator, but localstack doesn't like batch deletes :shrug:
  out, err := client.ListObjects(&s3.ListObjectsInput{
  Bucket: aws.String(name),
@@ -460,6 +468,8 @@ func deleteBucket(t testing.TB, client *s3.S3, name string) {
 }
 
 func readFile(t testing.TB, path string) string {
+ t.Helper()
+
  contents, err := ioutil.ReadFile(path)
  if err != nil {
  t.Fatal(err)
@@ -469,6 +479,8 @@ func readFile(t testing.TB, path string) string {
 }
 
 func download(t testing.TB, downloader *s3manager.Downloader, bucket string, key string) string {
+ t.Helper()
+
  b := &aws.WriteAtBuffer{}
  _, err := downloader.Download(b, &s3.GetObjectInput{
  Bucket: aws.String(bucket),

chiv_test.go

@@ -18,14 +18,14 @@ import (
 
 func TestArchiveRows(t *testing.T) {
  cases := []struct {
- name  string
- rows  *rows
- uploader  *uploader
- bucket  string
- formatter  *formatter
- formatterFuncErr error
- options  []chiv.Option
- expectedErr  string
+ name string
+ rows *rows
+ uploader *uploader
+ bucket string
+ formatter *formatter
+ formatErr  error
+ options []chiv.Option
+ expectedErr string
  }{
  {
  name: "base case",
@@ -72,10 +72,10 @@ func TestArchiveRows(t *testing.T) {
  columns: []string{"first_column", "second_column"},
  scan: [][]string{{"first", "second"}},
  },
- formatterFuncErr: errors.New("formatter func"),
- expectedErr:  "formatter func",
- uploader:  &uploader{},
- formatter:  &formatter{},
+ formatErr:  errors.New("formatter func"),
+ expectedErr: "formatter func",
+ uploader: &uploader{},
+ formatter: &formatter{},
  },
  {
  name: "scan error",
@@ -140,7 +140,7 @@ func TestArchiveRows(t *testing.T) {
  for _, test := range cases {
  t.Run(test.name, func(t *testing.T) {
  var (
- options = append(test.options, chiv.WithFormat(formatterFunc(test.formatter, test.formatterFuncErr)))
+ options = append(test.options, chiv.WithFormat(format(test.formatter, test.formatErr)))
  err = chiv.ArchiveRows(test.rows, test.uploader, "bucket", options...)
  )
 
@@ -173,7 +173,7 @@ type rows struct {
 }
 
 func (r *rows) ColumnTypes() ([]*sql.ColumnType, error) {
- return make([]*sql.ColumnType, len(r.columns)), r.columnTypesErr
+ return make([](*sql.ColumnType), len(r.columns)), r.columnTypesErr
 }
 
 func (r *rows) Next() bool {
@@ -215,8 +215,8 @@ func (u *uploader) UploadWithContext(ctx aws.Context, input *s3manager.UploadInp
  return nil, u.uploadErr
 }
 
-func formatterFunc(f chiv.Formatter, err error) chiv.FormatterFunc {
- return func(_ io.Writer, _ []*sql.ColumnType) (chiv.Formatter, error) {
+func format(f chiv.Formatter, err error) chiv.Format {
+ return func(_ io.Writer, _ []chiv.Column) (chiv.Formatter, error) {
  return f, err
  }
 }

cmd/chiv/main.go

@@ -136,7 +136,7 @@ func from(ctx *cli.Context) config {
  cfg.options = append(cfg.options, chiv.WithColumns(columns...))
  }
 
- var m = map[string]chiv.FormatterFunc{
+ var m = map[string]chiv.Format{
  "csv": chiv.CSV,
  "yaml": chiv.YAML,
  "json": chiv.JSON,

formatters.go

@@ -1,7 +1,6 @@
 package chiv
 
 import (
- "database/sql"
  "encoding/csv"
  "encoding/json"
  "errors"
@@ -13,8 +12,13 @@ import (
  yaml "gopkg.in/yaml.v2"
 )
 
-// FormatterFunc returns an initialized Formatter.
-type FormatterFunc func(io.Writer, []*sql.ColumnType) (Formatter, error)
+type Column interface {
+ DatabaseTypeName() string
+ Name() string
+}
+
+// Format is a function representing an upload format. It returns an initialized Formatter.
+type Format func(io.Writer, []Column) (Formatter, error)
 
 // Formatter formats and writes records.
 type Formatter interface {
@@ -30,7 +34,7 @@ type csvFormatter struct {
 }
 
 // CSV writes column headers and returns an initialized CSV formatter.
-func CSV(w io.Writer, columns []*sql.ColumnType) (Formatter, error) {
+func CSV(w io.Writer, columns []Column) (Formatter, error) {
  f := &csvFormatter{
  w: csv.NewWriter(w),
  count: len(columns),
@@ -74,11 +78,11 @@ func (f *csvFormatter) Close() error {
 
 type yamlFormatter struct {
  w io.Writer
- columns []*sql.ColumnType
+ columns []Column
 }
 
 // YAML returns an initialized YAML formatter.
-func YAML(w io.Writer, columns []*sql.ColumnType) (Formatter, error) {
+func YAML(w io.Writer, columns []Column) (Formatter, error) {
  f := yamlFormatter{
  w: w,
  columns: columns,
@@ -119,12 +123,12 @@ const (
 
 type jsonFormatter struct {
  w io.Writer
- columns []*sql.ColumnType
+ columns []Column
  notFirst bool
 }
 
 // JSON opens a JSON array and returns an initialized JSON formatter.
-func JSON(w io.Writer, columns []*sql.ColumnType) (Formatter, error) {
+func JSON(w io.Writer, columns []Column) (Formatter, error) {
  f := jsonFormatter{
  w: w,
  columns: columns,
@@ -207,7 +211,7 @@ func parse(v []byte, t string) (interface{}, error) {
  }
 }
 
-func buildMap(record [][]byte, columns []*sql.ColumnType) (map[string]interface{}, error) {
+func buildMap(record [][]byte, columns []Column) (map[string]interface{}, error) {
  m := make(map[string]interface{})
  for i, column := range columns {
  r, err := parse(record[i], column.DatabaseTypeName())

formatters_test.go

@@ -3,17 +3,117 @@
 package chiv_test
 
 import (
+ "bytes"
  "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "gavincabbage.com/chiv"
 )
 
+type testCase struct {
+ name string
+ columns []column
+ records [][][]byte
+}
+
+var cases = []testCase{
+ {
+ name: "base case",
+ columns: []column{
+ {
+ name: "first_column",
+ databaseType: "INTEGER",
+ },
+ {
+ name: "second_column",
+ databaseType: "TEXT",
+ },
+ },
+ records: [][][]byte{
+ {
+ []byte("1"),
+ []byte("first_row"),
+ },
+ {
+ []byte("2"),
+ []byte("second_row"),
+ },
+ {
+ []byte("3"),
+ []byte("third_row"),
+ },
+ },
+ },
+}
+
 func TestCsvFormatter(t *testing.T) {
- t.Skip()
+ expected := []string{
+ `first_column,second_column
+1,first_row
+2,second_row
+3,third_row
+`,
+ }
+
+ test(t, expected, chiv.CSV)
 }
 
 func TestYamlFormatter(t *testing.T) {
- t.Skip()
+ expected := []string{
+ `- first_column: 1
+ second_column: first_row
+- first_column: 2
+ second_column: second_row
+- first_column: 3
+ second_column: third_row
+`,
+ }
+
+ test(t, expected, chiv.YAML)
 }
 
 func TestJsonFormatter(t *testing.T) {
- t.Skip()
+ expected := []string{
+ `[{"first_column":1,"second_column":"first_row"},{"first_column":2,"second_column":"second_row"},{"first_column":3,"second_column":"third_row"}]`,
+ }
+
+ test(t, expected, chiv.JSON)
+}
+
+func test(t *testing.T, expected []string, format chiv.Format) {
+ for i, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ var (
+ b = bytes.Buffer{}
+ columns = make([]chiv.Column, len(test.columns))
+ )
+ for i := range test.columns {
+ columns[i] = test.columns[i]
+ }
+
+ subject, err := format(&b, columns)
+ assert.NoError(t, err)
+
+ for _, record := range test.records {
+ assert.NoError(t, subject.Format(record))
+ }
+
+ assert.NoError(t, subject.Close())
+ assert.Equal(t, expected[i], b.String())
+ })
+ }
+}
+
+type column struct {
+ databaseType string
+ name string
+}
+
+func (c column) DatabaseTypeName() string {
+ return c.databaseType
+}
+
+func (c column) Name() string {
+ return c.name
 }