From 8f3be7da790953e94f5b22456d499754202e2183 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Tue, 29 Nov 2022 18:42:22 +0200 Subject: [PATCH 1/5] enable SAT for redshift --- .../acceptance-test-config.yml | 17 ++++++++++++++++ .../integration_tests/basic_dataset.sql | 20 +++++++++++++++++++ .../integration_tests/configured_catalog.json | 19 ++++++++++++++++++ .../configured_catalog_inc.json | 16 +++++++++++++++ .../integration_tests/expected_records.json | 2 ++ 5 files changed, 74 insertions(+) create mode 100644 airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql create mode 100644 airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog_inc.json create mode 100644 airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json diff --git a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml index 269041c35e8f..ef98f6b3976c 100644 --- a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml @@ -5,3 +5,20 @@ tests: spec: - spec_path: "src/test-integration/resources/expected_spec.json" config_path: "secrets/config.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + timeout_seconds: "1200" + expect_records: + path: "integration_tests/expected_records.json" + full_refresh: + - config_path: "secrets/config.json" + timeout_seconds: "1200" + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_inc.json" + timeout_seconds: "1200" diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql b/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql new file mode 100644 index 000000000000..e1678611680e --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS sat_test_dataset.sat_basic_dataset ( + id INTEGER, + test_column_1 SMALLINT, + test_column_2 INTEGER, + test_column_3 BIGINT, + test_column_4 DECIMAL, + test_column_5 REAL, + test_column_6 DOUBLE PRECISION, + test_column_7 BOOLEAN, + test_column_8 CHAR, + test_column_9 VARCHAR, + test_column_10 DATE, + test_column_11 TIMESTAMP, + test_column_12 TIMESTAMPTZ, + test_column_13 TIME, + test_column_14 TIMETZ, + test_column_15 VARBYTE); + +insert into sat_test_dataset.sat_basic_dataset values (1, 1, 126, 1024, 555.666, 777.888, 999.000, true, 'q', 'some text', '2008-12-31', 'Jun 1,2008 09:59:59', 'Jun 1,2008 09:59:59 EST', '04:05:06', '04:05:06 EST', 'xxx'::varbyte); +insert into sat_test_dataset.sat_basic_dataset values (2, -5, -126, -1024, -555.666, -777.888, -999.000, false, 'g', 'new text', '1987-10-10', 'Jun 21,2005 12:00:59', 'Oct 15,2003 09:59:59 EST', '04:05:00', '04:05:00 EST', 'yyy'::varbyte); \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..b1153b2a245c --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog.json @@ -0,0 +1,19 @@ +{ + "streams": [ + { + "stream": { + "name": "sat_basic_dataset", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog_inc.json b/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog_inc.json new file mode 100644 index 000000000000..55458dbbfe40 --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog_inc.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "sat_basic_dataset", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json new file mode 100644 index 000000000000..1fc4b9afa423 --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json @@ -0,0 +1,2 @@ +{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31T00:00:00Z","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:06Z","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } +{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10T00:00:00Z","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:00Z","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } From 0c55000865c1850043a46f0a08d12c7ea62dd101 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 30 Nov 2022 18:04:28 +0200 Subject: [PATCH 2/5] add bigger timeouts + specify catalogs (default works wrong) --- .../connectors/source-redshift/acceptance-test-config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml index ef98f6b3976c..a9fb15a52790 100644 --- a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml @@ -4,19 +4,24 @@ connector_image: airbyte/source-redshift:dev tests: spec: - spec_path: "src/test-integration/resources/expected_spec.json" + timeout_seconds: "1200" config_path: "secrets/config.json" connection: - config_path: "secrets/config.json" + timeout_seconds: "1200" status: "succeed" discovery: - config_path: "secrets/config.json" + timeout_seconds: "1200" basic_read: - config_path: "secrets/config.json" timeout_seconds: "1200" + configured_catalog_path: "integration_tests/configured_catalog.json" expect_records: path: "integration_tests/expected_records.json" full_refresh: - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" timeout_seconds: "1200" incremental: - config_path: "secrets/config.json" From fe9266ff090cae3cbbc08cf6ab064c7eee996b61 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Fri, 2 Dec 2022 15:05:58 +0200 Subject: [PATCH 3/5] new line --- .../source-redshift/integration_tests/basic_dataset.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql b/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql index e1678611680e..743dc4076e97 100644 --- a/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql @@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS sat_test_dataset.sat_basic_dataset ( test_column_15 VARBYTE); insert into sat_test_dataset.sat_basic_dataset values (1, 1, 126, 1024, 555.666, 777.888, 999.000, true, 'q', 'some text', '2008-12-31', 'Jun 1,2008 09:59:59', 'Jun 1,2008 09:59:59 EST', '04:05:06', '04:05:06 EST', 'xxx'::varbyte); -insert into sat_test_dataset.sat_basic_dataset values (2, -5, -126, -1024, -555.666, -777.888, -999.000, false, 'g', 'new text', '1987-10-10', 'Jun 21,2005 12:00:59', 'Oct 15,2003 09:59:59 EST', '04:05:00', '04:05:00 EST', 'yyy'::varbyte); \ No newline at end of file +insert into sat_test_dataset.sat_basic_dataset values (2, -5, -126, -1024, -555.666, -777.888, -999.000, false, 'g', 'new text', '1987-10-10', 'Jun 21,2005 12:00:59', 'Oct 15,2003 09:59:59 EST', '04:05:00', '04:05:00 EST', 'yyy'::varbyte); From c8a65a19dbbba9f0c736e1d72dc73b1d474587d6 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 7 Dec 2022 16:16:56 +0200 Subject: [PATCH 4/5] readme --- .../connectors/source-redshift/integration_tests/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 airbyte-integrations/connectors/source-redshift/integration_tests/README.md diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/README.md b/airbyte-integrations/connectors/source-redshift/integration_tests/README.md new file mode 100644 index 000000000000..0da59b2db5f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/README.md @@ -0,0 +1,3 @@ +# Seeding the dataset +You can find the SQL scripts in this folder if you need to create or fix the SAT dataset. +For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy). \ No newline at end of file From b6d3ec18bf95e300d11f7508d8e687f7d43a8a74 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 7 Dec 2022 16:32:52 +0200 Subject: [PATCH 5/5] new format --- .../acceptance-test-config.yml | 46 +++++++++++-------- .../integration_tests/README.md | 2 +- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml index a9fb15a52790..56a9b1324f0a 100644 --- a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml @@ -1,29 +1,35 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-redshift:dev -tests: +acceptance_tests: spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - timeout_seconds: "1200" - config_path: "secrets/config.json" + tests: + - spec_path: "src/test-integration/resources/expected_spec.json" + timeout_seconds: "1200" + config_path: "secrets/config.json" connection: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - status: "succeed" + tests: + - config_path: "secrets/config.json" + timeout_seconds: "1200" + status: "succeed" discovery: - - config_path: "secrets/config.json" - timeout_seconds: "1200" + tests: + - config_path: "secrets/config.json" + timeout_seconds: "1200" basic_read: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.json" + tests: + - config_path: "secrets/config.json" + timeout_seconds: "1200" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_records: + path: "integration_tests/expected_records.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: "1200" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: "1200" incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_inc.json" - timeout_seconds: "1200" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_inc.json" + timeout_seconds: "1200" diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/README.md b/airbyte-integrations/connectors/source-redshift/integration_tests/README.md index 0da59b2db5f2..96aa5492669b 100644 --- a/airbyte-integrations/connectors/source-redshift/integration_tests/README.md +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/README.md @@ -1,3 +1,3 @@ # Seeding the dataset You can find the SQL scripts in this folder if you need to create or fix the SAT dataset. -For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy). \ No newline at end of file +For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy).