diff --git a/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml index d634ced570a5..b893dfbbb071 100644 --- a/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml @@ -1,7 +1,29 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-bigquery:dev -tests: +acceptance_tests: spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" + tests: + - spec_path: "src/test-integration/resources/expected_spec.json" + config_path: "src/test-integration/resources/dummy_config.json" + connection: + tests: + - config_path: "secrets/sat-config.json" + status: "succeed" + discovery: + tests: + - config_path: "secrets/sat-config.json" + basic_read: + tests: + - config_path: "secrets/sat-config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_records: + path: "integration_tests/expected_records.json" + full_refresh: + tests: + - config_path: "secrets/sat-config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + tests: + - config_path: "secrets/sat-config.json" + configured_catalog_path: "integration_tests/configured_catalog_inc.json" diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/README.md b/airbyte-integrations/connectors/source-bigquery/integration_tests/README.md new file mode 100644 index 000000000000..96aa5492669b --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/README.md @@ -0,0 +1,3 @@ +# Seeding the dataset +You can find the SQL scripts in this folder if you need to create or fix the SAT dataset. +For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy). diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/basic_dataset.sql b/airbyte-integrations/connectors/source-bigquery/integration_tests/basic_dataset.sql new file mode 100644 index 000000000000..b87f8a5efb08 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/basic_dataset.sql @@ -0,0 +1,33 @@ +CREATE TABLE + `sat_test_dataset.sat_basic_dataset`(id NUMERIC(29), + test_column_1 int64, + test_column_10 decimal(29, + 9), + test_column_11 bigdecimal(76, + 38), + test_column_12 float64, + test_column_13 bool, + test_column_15 date, + test_column_16 datetime, + test_column_17 timestamp, + test_column_19 string, + test_column_2 int, + test_column_21 time, + test_column_3 smallint, + test_column_4 integer, + test_column_5 bigint, + test_column_6 tinyint, + test_column_7 byteint, + test_column_8 numeric(29, + 9), + test_column_9 bignumeric(76, + 38) ); + + +INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (1, -128, -128, -128, -128, true, date('2021-10-20'), datetime('2021-10-20 11:22:33'), timestamp('2021-10-20 11:22:33'), 'qwe', -128, TIME(15, 30, 00), -128, -128, -128, -128, -128, -128, -128); +INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (2, 127, 127, 127, 127, false, date('9999-12-31'), datetime('9999-12-31 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 127, 127); +INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (3, 9223372036854775807, 999999999999999999, 999999999999999999, 0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 999999999999999999, 999999999999999999); +INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (4, -9223372036854775808, -999999999999999999, -999999999999999999, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -999999999999999999, -999999999999999999); +INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (5, -9223372036854775808, 0.123456789, 0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 0.123456789, 0.123456789); +INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (6, -9223372036854775808, -0.123456789, -0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -0.123456789, -0.123456789); +INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (7, -9223372036854775808, -0.123456789, -0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -0.123456789, -0.123456789); diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..b1153b2a245c --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog.json @@ -0,0 +1,19 @@ +{ + "streams": [ + { + "stream": { + "name": "sat_basic_dataset", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog_inc.json b/airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog_inc.json new file mode 100644 index 000000000000..55458dbbfe40 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog_inc.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "sat_basic_dataset", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/expected_records.json b/airbyte-integrations/connectors/source-bigquery/integration_tests/expected_records.json new file mode 100644 index 000000000000..9e3198b3e2b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/expected_records.json @@ -0,0 +1,7 @@ +{"stream": "sat_basic_dataset", "data": {"id": 1, "test_column_1": -128, "test_column_10": -128, "test_column_11": -128, "test_column_12": -128.0, "test_column_13": true, "test_column_15": "2021-10-20T00:00:00Z", "test_column_16": "2021-10-20T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "qwe", "test_column_2": -128, "test_column_21": "15:30:00", "test_column_3": -128, "test_column_4": -128, "test_column_5": -128, "test_column_6": -128, "test_column_7": -128, "test_column_8": -128, "test_column_9": -128 }, "emitted_at": 1669643054045 } +{"stream": "sat_basic_dataset", "data": {"id": 2, "test_column_1": 127, "test_column_10": 127, "test_column_11": 127, "test_column_12": 127.0, "test_column_13": false, "test_column_15": "9999-12-31T00:00:00Z", "test_column_16": "9999-12-31T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 127, "test_column_9": 127 }, "emitted_at": 1669643054045 } +{"stream": "sat_basic_dataset", "data": {"id": 3, "test_column_1": 9223372036854775807, "test_column_10": 999999999999999999, "test_column_11": 999999999999999999, "test_column_12": 0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 999999999999999999, "test_column_9": 999999999999999999 }, "emitted_at": 1669643054045 } +{"stream": "sat_basic_dataset", "data": {"id": 7, "test_column_1": -9223372036854775808, "test_column_10": -0.123456789, "test_column_11": -0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -0.123456789, "test_column_9": -0.123456789 }, "emitted_at": 1669643054045 } +{"stream": "sat_basic_dataset", "data": {"id": 4, "test_column_1": -9223372036854775808, "test_column_10": -999999999999999999, "test_column_11": -999999999999999999, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -999999999999999999, "test_column_9": -999999999999999999 }, "emitted_at": 1669643054045 } +{"stream": "sat_basic_dataset", "data": {"id": 5, "test_column_1": -9223372036854775808, "test_column_10": 0.123456789, "test_column_11": 0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 0.123456789, "test_column_9": 0.123456789 }, "emitted_at": 1669643054045 } +{"stream": "sat_basic_dataset", "data": {"id": 6, "test_column_1": -9223372036854775808, "test_column_10": -0.123456789, "test_column_11": -0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -0.123456789, "test_column_9": -0.123456789 }, "emitted_at": 1669643054045 } diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/full_dataset.sql b/airbyte-integrations/connectors/source-bigquery/integration_tests/full_dataset.sql new file mode 100644 index 000000000000..2bae50237ede --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/full_dataset.sql @@ -0,0 +1,47 @@ +CREATE TABLE + sat_test_dataset.sat_full_dataset(id NUMERIC(29), + test_column_1 int64, + test_column_10 decimal(29, + 9), + test_column_11 bigdecimal(76, + 38), + test_column_12 float64, + test_column_13 bool, + test_column_14 bytes, + test_column_15 date, + test_column_16 datetime, + test_column_17 timestamp, + test_column_18 geography, + test_column_19 string, + test_column_2 int, + test_column_20 STRUCT, + test_column_21 time, + test_column_22 ARRAY, + test_column_23 STRUCT>>, + test_column_24 ARRAY>, + test_column_25 ARRAY>>>, + test_column_26 INTERVAL, + test_column_3 smallint, + test_column_4 integer, + test_column_5 bigint, + test_column_6 tinyint, + test_column_7 byteint, + test_column_8 numeric(29, + 9), + test_column_9 bignumeric(76, + 38) ); + +INSERT INTO sat_test_dataset.sat_full_dataset VALUES (1, null, null, null, null, true, FROM_BASE64("test"), date('2021-10-20'), datetime('2021-10-20 11:22:33'), timestamp('2021-10-20 11:22:33'), ST_GEOGFROMTEXT('POINT(1 2)'), 'qwe', null, STRUCT("B.A",12), TIME(15, 30, 00), ['a', 'b'], STRUCT('s' as frst, 1 as sec, STRUCT(555 as id_col, STRUCT(TIME(15, 30, 00) as time) as mega_obbj) as obbj), [STRUCT('qqq' as fff, 1 as ggg), STRUCT('kkk' as fff, 2 as ggg)], [STRUCT('qqq' as fff, [STRUCT('fff' as ooo, 1 as kkk), STRUCT('hhh' as ooo, 2 as kkk)] as ggg)], MAKE_INTERVAL(2021, 10, 10, 10, 10, 10), null, null, null, null, null, null, null); +INSERT INTO sat_test_dataset.sat_full_dataset VALUES (2, -128, -128, -128, -128, false, null, date('9999-12-31'), datetime('9999-12-31 11:22:33'), null, null, 'йцу', -128, null, null, null, null, null, null, null, -128, -128, -128, -128, -128, -128, -128); +INSERT INTO sat_test_dataset.sat_full_dataset VALUES (3, 127, 127, 127, 127, null, null, date('0001-01-01'), datetime('0001-01-01 11:22:33'), null, null, null, 127, null, null, null, null, null, null, null, 127, 127, 127, 127, 127, 127, 127); +INSERT INTO sat_test_dataset.sat_full_dataset VALUES (4, 9223372036854775807, 999999999999999999, 999999999999999999, 0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 999999999999999999, 999999999999999999); +INSERT INTO sat_test_dataset.sat_full_dataset VALUES (5, -9223372036854775808, -999999999999999999, -999999999999999999, -0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, -999999999999999999, -999999999999999999); +INSERT INTO sat_test_dataset.sat_full_dataset VALUES (6, null, 0.123456789, 0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0.123456789, 0.123456789); +INSERT INTO sat_test_dataset.sat_full_dataset VALUES (7, null, -0.123456789, -0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, -0.123456789, -0.123456789);