tests

gavincabbage · Apr 1, 2019 · 145bceb · 145bceb
1 parent 92eccee
commit 145bceb
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 [![GoDoc](https://godoc.org/github.com/gavincabbage/chiv?status.svg)](https://godoc.org/github.com/gavincabbage/chiv)
 [![License](http://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/gavincabbage/chiv/blob/master/LICENSE)
 
-Archive arbitrarily large relational database tables to Amazon S3.
+Archive relational database tables to Amazon S3.
 
 ## Example
 

diff --git a/chiv.go b/chiv.go
@@ -89,7 +89,7 @@ func (a *archiver) archive(table string, bucket string) error {
 
 type formatter interface {
  Begin([]*sql.ColumnType) error
- Write([]sql.RawBytes) error
+ Write([][]byte) error
  End() error
 }
 
@@ -125,26 +125,29 @@ func (a *archiver) download(wc io.WriteCloser, table string, errs chan error) {
 
  var (
  rawBytes = make([]sql.RawBytes, len(columns))
- record = make([]interface{}, len(columns))
+ scanned = make([]interface{}, len(columns))
+ record = make([][]byte, len(columns))
  )
  for i := range rawBytes {
- record[i] = &rawBytes[i]
+ scanned[i] = &rawBytes[i]
  }
 
  for rows.Next() {
- err = rows.Scan(record...)
+ err = rows.Scan(scanned...)
  if err != nil {
  errs <- err
  return
  }
 
  for i, raw := range rawBytes {
  if raw == nil && a.config.null != nil {
- rawBytes[i] = a.config.null
+ record[i] = a.config.null
+ } else {
+ record[i] = raw
  }
  }
 
- if err := w.Write(rawBytes); err != nil {
+ if err := w.Write(record); err != nil {
  errs <- err
  return
  }

diff --git a/formatters.go b/formatters.go
@@ -30,15 +30,15 @@ type csvFormatter struct {
 func (c *csvFormatter) Begin(columns []*sql.ColumnType) error {
  c.count = len(columns)
 
- header := make([]string, 0, c.count)
- for _, column := range columns {
- header = append(header, column.Name())
+ header := make([]string, c.count)
+ for i, column := range columns {
+ header[i] = column.Name()
  }
 
  return c.w.Write(header)
 }
 
-func (c *csvFormatter) Write(record []sql.RawBytes) error {
+func (c *csvFormatter) Write(record [][]byte) error {
  if c.count != len(record) {
  return ErrRecordLength
  }
@@ -65,7 +65,7 @@ func (c *yamlFormatter) Begin(columns []*sql.ColumnType) error {
  return nil
 }
 
-func (c *yamlFormatter) Write(record []sql.RawBytes) error {
+func (c *yamlFormatter) Write(record [][]byte) error {
  if len(c.columns) != len(record) {
  return ErrRecordLength
  }
@@ -89,7 +89,7 @@ func (c *jsonFormatter) Begin(columns []*sql.ColumnType) error {
  return writeByte(c.w, openBracket)
 }
 
-func (c *jsonFormatter) Write(record []sql.RawBytes) error {
+func (c *jsonFormatter) Write(record [][]byte) error {
  if len(c.columns) != len(record) {
  return ErrRecordLength
  }
@@ -141,7 +141,7 @@ func writeByte(w io.Writer, b byte) error {
  return nil
 }
 
-func parse(b sql.RawBytes, t string) (interface{}, error) {
+func parse(b []byte, t string) (interface{}, error) {
  if b == nil {
  return nil, nil
  }
@@ -150,7 +150,7 @@ func parse(b sql.RawBytes, t string) (interface{}, error) {
  s = string(b)
  boolRegex = regexp.MustCompile("BOOL*")
  intRegex = regexp.MustCompile("INT*")
- decimalRegex = regexp.MustCompile("DECIMAL*")
+ decimalRegex = regexp.MustCompile("DECIMAL*|FLOAT*|NUMERIC*")
  )
  switch {
  case boolRegex.MatchString(t):

diff --git a/testdata/postgres.csv b/testdata/postgres.csv
@@ -1,4 +1,4 @@
-id,text_column,char_column,int_column,bool_column,ts_column
-ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,true,2018-01-04T00:00:00Z
-4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,,100,true,2018-02-04T00:00:00Z
-7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,false,2018-02-05T00:00:00Z
+id,text_column,char_column,int_column,float_column,bool_column,ts_column
+ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,3.14,true,2018-01-04T00:00:00Z
+4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,,100,3.141592,true,2018-02-04T00:00:00Z
+7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,,false,2018-02-05T00:00:00Z
diff --git a/testdata/postgres.json b/testdata/postgres.json
@@ -1 +1 @@
-[{"bool_column":true,"char_column":"some chars","id":"ea09d13c-f441-4550-9492-115f8b409c96","int_column":42,"text_column":"some text","ts_column":"2018-01-04T00:00:00Z"},{"bool_column":true,"char_column":null,"id":"4289a9e3-32d5-4bad-b79b-034c528e8f41","int_column":100,"text_column":"some other text","ts_column":"2018-02-04T00:00:00Z"},{"bool_column":false,"char_column":"some more chars","id":"7530a381-526a-42aa-a9ba-97fb2bca283f","int_column":101,"text_column":"some more text","ts_column":"2018-02-05T00:00:00Z"}]
+[{"bool_column":true,"char_column":"some chars","float_column":3.14,"id":"ea09d13c-f441-4550-9492-115f8b409c96","int_column":42,"text_column":"some text","ts_column":"2018-01-04T00:00:00Z"},{"bool_column":true,"char_column":null,"float_column":3.141592,"id":"4289a9e3-32d5-4bad-b79b-034c528e8f41","int_column":100,"text_column":"some other text","ts_column":"2018-02-04T00:00:00Z"},{"bool_column":false,"char_column":"some more chars","float_column":null,"id":"7530a381-526a-42aa-a9ba-97fb2bca283f","int_column":101,"text_column":"some more text","ts_column":"2018-02-05T00:00:00Z"}]
diff --git a/testdata/postgres_setup.sql b/testdata/postgres_setup.sql
@@ -1,35 +1,39 @@
 CREATE TABLE IF NOT EXISTS "postgres_table" (
- id UUID PRIMARY KEY,
- text_column TEXT,
- char_column VARCHAR(50),
- int_column INTEGER,
- bool_column BOOLEAN,
- ts_column TIMESTAMP
+ id UUID PRIMARY KEY,
+ text_column TEXT,
+ char_column VARCHAR(50),
+ int_column INTEGER,
+ float_column NUMERIC,
+ bool_column BOOLEAN,
+ ts_column TIMESTAMP
 );
 
 INSERT INTO "postgres_table" VALUES (
- 'ea09d13c-f441-4550-9492-115f8b409c96',
- 'some text',
- 'some chars',
- 42,
- true,
- '2018-01-04'::timestamp
+ 'ea09d13c-f441-4550-9492-115f8b409c96',
+ 'some text',
+ 'some chars',
+ 42,
+ 3.14,
+ true,
+ '2018-01-04'::timestamp
 );
 
 INSERT INTO "postgres_table" VALUES (
- '4289a9e3-32d5-4bad-b79b-034c528e8f41',
- 'some other text',
- null,
- 100,
- true,
- '2018-02-04'::timestamp
+ '4289a9e3-32d5-4bad-b79b-034c528e8f41',
+ 'some other text',
+ null,
+ 100,
+ 3.141592,
+ true,
+ '2018-02-04'::timestamp
  );
 
 INSERT INTO "postgres_table" VALUES (
- '7530a381-526a-42aa-a9ba-97fb2bca283f',
- 'some more text',
- 'some more chars',
- 101,
- false,
- '2018-02-05'::timestamp
+ '7530a381-526a-42aa-a9ba-97fb2bca283f',
+ 'some more text',
+ 'some more chars',
+ 101,
+ null,
+ false,
+ '2018-02-05'::timestamp
 );
diff --git a/testdata/postgres_with_null.csv b/testdata/postgres_with_null.csv
@@ -1,4 +1,4 @@
-id,text_column,char_column,int_column,bool_column,ts_column
-ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,true,2018-01-04T00:00:00Z
-4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,custom_null,100,true,2018-02-04T00:00:00Z
-7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,false,2018-02-05T00:00:00Z
+id,text_column,char_column,int_column,float_column,bool_column,ts_column
+ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,3.14,true,2018-01-04T00:00:00Z
+4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,custom_null,100,3.141592,true,2018-02-04T00:00:00Z
+7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,custom_null,false,2018-02-05T00:00:00Z