From 7869254cfe82bd206905b51f47be7e0c230049d5 Mon Sep 17 00:00:00 2001 From: Gavin Cabbage Date: Tue, 1 Oct 2019 00:22:28 -0400 Subject: [PATCH] test postgres json column --- .github/workflows/build.yml | 4 ++-- README.md | 2 +- chiv_formatters_test.go | 30 ++++++++++++++++-------- chiv_integration_test.go | 17 -------------- testdata/postgres/postgres.csv | 8 +++---- testdata/postgres/postgres.json | 2 +- testdata/postgres/postgres.yaml | 3 +++ testdata/postgres/postgres_setup.sql | 12 ++++++---- testdata/postgres/postgres_with_null.csv | 8 +++---- 9 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3187485..64546a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,7 +40,7 @@ jobs: - name: Install dependencies run: go mod download - name: Run unit tests - run: go test -v -p 1 -tags=unit -covermode=atomic -timeout=30s ./... + run: go test -v -p 1 -tags=unit -timeout=30s ./... benchmark: name: Benchmark @@ -101,7 +101,7 @@ jobs: - name: Install dependencies run: go mod download - name: Run integration tests - run: go test -v -p 1 -tags=integration -covermode=atomic -timeout=60s ./... + run: go test -v -p 1 -tags=integration -timeout=60s ./... env: AWS_ACCESS_KEY_ID: bogus AWS_SECRET_KEY: bogus diff --git a/README.md b/README.md index de31cb7..c7b59b5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ---

- Archive relational database tables to Amazon S3. + Archive relational data to Amazon S3.

diff --git a/chiv_formatters_test.go b/chiv_formatters_test.go index 7ecbdc5..b1fd337 100644 --- a/chiv_formatters_test.go +++ b/chiv_formatters_test.go @@ -29,30 +29,37 @@ var cases = []testCase{ name: "second_column", databaseType: "TEXT", }, + { + name: "third_column", + databaseType: "FLOAT", + }, }, records: [][][]byte{ { []byte("1"), []byte("first_row"), + []byte("100"), }, { []byte("2"), []byte("second_row"), + []byte("12.12"), }, { []byte("3"), []byte("third_row"), + []byte("42.42"), }, }, }, } func TestCsvFormatter(t *testing.T) { - expected := []string{ - `first_column,second_column -1,first_row -2,second_row -3,third_row + expected := []string{` +first_column,second_column,third_column +1,first_row,100 +2,second_row,12.12 +3,third_row,42.42 `, } @@ -60,13 +67,16 @@ func TestCsvFormatter(t *testing.T) { } func TestYamlFormatter(t *testing.T) { - expected := []string{ - `- first_column: 1 + expected := []string{` +- first_column: 1 second_column: first_row + third_column: 100 - first_column: 2 second_column: second_row + third_column: 12.12 - first_column: 3 second_column: third_row + third_column: 42.42 `, } @@ -74,8 +84,8 @@ func TestYamlFormatter(t *testing.T) { } func TestJsonFormatter(t *testing.T) { - expected := []string{ - `[{"first_column":1,"second_column":"first_row"},{"first_column":2,"second_column":"second_row"},{"first_column":3,"second_column":"third_row"}]`, + expected := []string{` +[{"first_column":1,"second_column":"first_row","third_column":100},{"first_column":2,"second_column":"second_row","third_column":12.12},{"first_column":3,"second_column":"third_row","third_column":42.42}]`, } test(t, expected, chiv.JSON) @@ -100,7 +110,7 @@ func test(t *testing.T, expected []string, format chiv.FormatterFunc) { } assert.NoError(t, subject.Close()) - assert.Equal(t, expected[i], b.String()) + assert.Equal(t, expected[i][1:], b.String()) }) } } diff --git a/chiv_integration_test.go b/chiv_integration_test.go index 1f155dc..ac6faa0 100644 --- a/chiv_integration_test.go +++ b/chiv_integration_test.go @@ -44,23 +44,6 @@ func TestArchiver_Archive(t *testing.T) { options []chiv.Option calls []call }{ - { - name: "happy path csv", - driver: "postgres", - database: os.Getenv("POSTGRES_URL"), - setup: "./testdata/postgres/postgres_setup.sql", - teardown: "./testdata/postgres/postgres_teardown.sql", - bucket: "test_bucket", - options: []chiv.Option{}, - calls: []call{ - { - expected: "./testdata/postgres/postgres.csv", - table: "postgres_table", - key: "postgres_table", - options: []chiv.Option{}, - }, - }, - }, { name: "postgres to csv", driver: "postgres", diff --git a/testdata/postgres/postgres.csv b/testdata/postgres/postgres.csv index 9538363..3577b1f 100644 --- a/testdata/postgres/postgres.csv +++ b/testdata/postgres/postgres.csv @@ -1,4 +1,4 @@ -id,text_column,char_column,int_column,float_column,bool_column,ts_column -ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,3.14,true,2018-01-04T00:00:00Z -4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,,100,3.141592,true,2018-02-04T00:00:00Z -7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,,false,2018-02-05T00:00:00Z +id,text_column,char_column,int_column,float_column,bool_column,ts_column,json_column +ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,3.14,true,2018-01-04T00:00:00Z,"{""key"":""value"",""num"":42}" +4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,,100,3.141592,true,2018-02-04T00:00:00Z,"{""other"":""value""}" +7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,,false,2018-02-05T00:00:00Z,"[{""item"":""in an array""},{""num"":999}]" diff --git a/testdata/postgres/postgres.json b/testdata/postgres/postgres.json index 6b9ed3b..a73edef 100644 --- a/testdata/postgres/postgres.json +++ b/testdata/postgres/postgres.json @@ -1 +1 @@ -[{"bool_column":true,"char_column":"some chars","float_column":3.14,"id":"ea09d13c-f441-4550-9492-115f8b409c96","int_column":42,"text_column":"some text","ts_column":"2018-01-04T00:00:00Z"},{"bool_column":true,"char_column":null,"float_column":3.141592,"id":"4289a9e3-32d5-4bad-b79b-034c528e8f41","int_column":100,"text_column":"some other text","ts_column":"2018-02-04T00:00:00Z"},{"bool_column":false,"char_column":"some more chars","float_column":null,"id":"7530a381-526a-42aa-a9ba-97fb2bca283f","int_column":101,"text_column":"some more text","ts_column":"2018-02-05T00:00:00Z"}] \ No newline at end of file +[{"bool_column":true,"char_column":"some chars","float_column":3.14,"id":"ea09d13c-f441-4550-9492-115f8b409c96","int_column":42,"json_column":"{\"key\":\"value\",\"num\":42}","text_column":"some text","ts_column":"2018-01-04T00:00:00Z"},{"bool_column":true,"char_column":null,"float_column":3.141592,"id":"4289a9e3-32d5-4bad-b79b-034c528e8f41","int_column":100,"json_column":"{\"other\":\"value\"}","text_column":"some other text","ts_column":"2018-02-04T00:00:00Z"},{"bool_column":false,"char_column":"some more chars","float_column":null,"id":"7530a381-526a-42aa-a9ba-97fb2bca283f","int_column":101,"json_column":"[{\"item\":\"in an array\"},{\"num\":999}]","text_column":"some more text","ts_column":"2018-02-05T00:00:00Z"}] \ No newline at end of file diff --git a/testdata/postgres/postgres.yaml b/testdata/postgres/postgres.yaml index 7ddb9b4..717208c 100644 --- a/testdata/postgres/postgres.yaml +++ b/testdata/postgres/postgres.yaml @@ -3,6 +3,7 @@ float_column: 3.14 id: ea09d13c-f441-4550-9492-115f8b409c96 int_column: 42 + json_column: '{"key":"value","num":42}' text_column: some text ts_column: "2018-01-04T00:00:00Z" - bool_column: true @@ -10,6 +11,7 @@ float_column: 3.141592 id: 4289a9e3-32d5-4bad-b79b-034c528e8f41 int_column: 100 + json_column: '{"other":"value"}' text_column: some other text ts_column: "2018-02-04T00:00:00Z" - bool_column: false @@ -17,5 +19,6 @@ float_column: null id: 7530a381-526a-42aa-a9ba-97fb2bca283f int_column: 101 + json_column: '[{"item":"in an array"},{"num":999}]' text_column: some more text ts_column: "2018-02-05T00:00:00Z" diff --git a/testdata/postgres/postgres_setup.sql b/testdata/postgres/postgres_setup.sql index 8588747..628c630 100644 --- a/testdata/postgres/postgres_setup.sql +++ b/testdata/postgres/postgres_setup.sql @@ -5,7 +5,8 @@ CREATE TABLE IF NOT EXISTS postgres_table ( int_column INTEGER, float_column NUMERIC, bool_column BOOLEAN, - ts_column TIMESTAMP + ts_column TIMESTAMP, + json_column JSON ); INSERT INTO postgres_table VALUES ( @@ -15,7 +16,8 @@ INSERT INTO postgres_table VALUES ( 42, 3.14, true, - '2018-01-04'::timestamp + '2018-01-04'::timestamp, + '{"key":"value","num":42}' ); INSERT INTO postgres_table VALUES ( @@ -25,7 +27,8 @@ INSERT INTO postgres_table VALUES ( 100, 3.141592, true, - '2018-02-04'::timestamp + '2018-02-04'::timestamp, + '{"other":"value"}' ); INSERT INTO postgres_table VALUES ( @@ -35,5 +38,6 @@ INSERT INTO postgres_table VALUES ( 101, null, false, - '2018-02-05'::timestamp + '2018-02-05'::timestamp, + '[{"item":"in an array"},{"num":999}]' ); \ No newline at end of file diff --git a/testdata/postgres/postgres_with_null.csv b/testdata/postgres/postgres_with_null.csv index 4d6ee9a..ebaf3ef 100644 --- a/testdata/postgres/postgres_with_null.csv +++ b/testdata/postgres/postgres_with_null.csv @@ -1,4 +1,4 @@ -id,text_column,char_column,int_column,float_column,bool_column,ts_column -ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,3.14,true,2018-01-04T00:00:00Z -4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,custom_null,100,3.141592,true,2018-02-04T00:00:00Z -7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,custom_null,false,2018-02-05T00:00:00Z +id,text_column,char_column,int_column,float_column,bool_column,ts_column,json_column +ea09d13c-f441-4550-9492-115f8b409c96,some text,some chars,42,3.14,true,2018-01-04T00:00:00Z,"{""key"":""value"",""num"":42}" +4289a9e3-32d5-4bad-b79b-034c528e8f41,some other text,custom_null,100,3.141592,true,2018-02-04T00:00:00Z,"{""other"":""value""}" +7530a381-526a-42aa-a9ba-97fb2bca283f,some more text,some more chars,101,custom_null,false,2018-02-05T00:00:00Z,"[{""item"":""in an array""},{""num"":999}]"