-
Notifications
You must be signed in to change notification settings - Fork 4.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable full SAT for the BigQuery source (#19720)
* enable full SAT for the BigQuery source * use basic database instead of full due to SAT fails * add full dataset sql * minor changes * specify catalogs (default works wrong) * new line * readme + format Co-authored-by: Augustin <augustin@airbyte.io>
- Loading branch information
1 parent
9eb0f0d
commit 36d4c92
Showing
7 changed files
with
150 additions
and
3 deletions.
There are no files selected for viewing
28 changes: 25 additions & 3 deletions
28
airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,29 @@ | ||
| # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) | ||
| # for more information about how to configure these tests | ||
| connector_image: airbyte/source-bigquery:dev | ||
| tests: | ||
| acceptance_tests: | ||
| spec: | ||
| - spec_path: "src/test-integration/resources/expected_spec.json" | ||
| config_path: "src/test-integration/resources/dummy_config.json" | ||
| tests: | ||
| - spec_path: "src/test-integration/resources/expected_spec.json" | ||
| config_path: "src/test-integration/resources/dummy_config.json" | ||
| connection: | ||
| tests: | ||
| - config_path: "secrets/sat-config.json" | ||
| status: "succeed" | ||
| discovery: | ||
| tests: | ||
| - config_path: "secrets/sat-config.json" | ||
| basic_read: | ||
| tests: | ||
| - config_path: "secrets/sat-config.json" | ||
| configured_catalog_path: "integration_tests/configured_catalog.json" | ||
| expect_records: | ||
| path: "integration_tests/expected_records.json" | ||
| full_refresh: | ||
| tests: | ||
| - config_path: "secrets/sat-config.json" | ||
| configured_catalog_path: "integration_tests/configured_catalog.json" | ||
| incremental: | ||
| tests: | ||
| - config_path: "secrets/sat-config.json" | ||
| configured_catalog_path: "integration_tests/configured_catalog_inc.json" |
3 changes: 3 additions & 0 deletions
3
airbyte-integrations/connectors/source-bigquery/integration_tests/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| # Seeding the dataset | ||
| You can find the SQL scripts in this folder if you need to create or fix the SAT dataset. | ||
| For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy). |
33 changes: 33 additions & 0 deletions
33
airbyte-integrations/connectors/source-bigquery/integration_tests/basic_dataset.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| CREATE TABLE | ||
| `sat_test_dataset.sat_basic_dataset`(id NUMERIC(29), | ||
| test_column_1 int64, | ||
| test_column_10 decimal(29, | ||
| 9), | ||
| test_column_11 bigdecimal(76, | ||
| 38), | ||
| test_column_12 float64, | ||
| test_column_13 bool, | ||
| test_column_15 date, | ||
| test_column_16 datetime, | ||
| test_column_17 timestamp, | ||
| test_column_19 string, | ||
| test_column_2 int, | ||
| test_column_21 time, | ||
| test_column_3 smallint, | ||
| test_column_4 integer, | ||
| test_column_5 bigint, | ||
| test_column_6 tinyint, | ||
| test_column_7 byteint, | ||
| test_column_8 numeric(29, | ||
| 9), | ||
| test_column_9 bignumeric(76, | ||
| 38) ); | ||
|
|
||
|
|
||
| INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (1, -128, -128, -128, -128, true, date('2021-10-20'), datetime('2021-10-20 11:22:33'), timestamp('2021-10-20 11:22:33'), 'qwe', -128, TIME(15, 30, 00), -128, -128, -128, -128, -128, -128, -128); | ||
| INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (2, 127, 127, 127, 127, false, date('9999-12-31'), datetime('9999-12-31 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 127, 127); | ||
| INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (3, 9223372036854775807, 999999999999999999, 999999999999999999, 0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 999999999999999999, 999999999999999999); | ||
| INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (4, -9223372036854775808, -999999999999999999, -999999999999999999, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -999999999999999999, -999999999999999999); | ||
| INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (5, -9223372036854775808, 0.123456789, 0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 0.123456789, 0.123456789); | ||
| INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (6, -9223372036854775808, -0.123456789, -0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -0.123456789, -0.123456789); | ||
| INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (7, -9223372036854775808, -0.123456789, -0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -0.123456789, -0.123456789); |
19 changes: 19 additions & 0 deletions
19
airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| { | ||
| "streams": [ | ||
| { | ||
| "stream": { | ||
| "name": "sat_basic_dataset", | ||
| "json_schema": {}, | ||
| "supported_sync_modes": ["full_refresh"], | ||
| "source_defined_cursor": null, | ||
| "default_cursor_field": null, | ||
| "source_defined_primary_key": [["id"]], | ||
| "namespace": null | ||
| }, | ||
| "sync_mode": "full_refresh", | ||
| "cursor_field": null, | ||
| "destination_sync_mode": "append", | ||
| "primary_key": null | ||
| } | ||
| ] | ||
| } |
16 changes: 16 additions & 0 deletions
16
...yte-integrations/connectors/source-bigquery/integration_tests/configured_catalog_inc.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| { | ||
| "streams": [ | ||
| { | ||
| "stream": { | ||
| "name": "sat_basic_dataset", | ||
| "json_schema": {}, | ||
| "supported_sync_modes": ["incremental"], | ||
| "source_defined_cursor": true, | ||
| "default_cursor_field": ["id"] | ||
| }, | ||
| "sync_mode": "incremental", | ||
| "cursor_field": ["id"], | ||
| "destination_sync_mode": "append" | ||
| } | ||
| ] | ||
| } |
7 changes: 7 additions & 0 deletions
7
airbyte-integrations/connectors/source-bigquery/integration_tests/expected_records.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| {"stream": "sat_basic_dataset", "data": {"id": 1, "test_column_1": -128, "test_column_10": -128, "test_column_11": -128, "test_column_12": -128.0, "test_column_13": true, "test_column_15": "2021-10-20T00:00:00Z", "test_column_16": "2021-10-20T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "qwe", "test_column_2": -128, "test_column_21": "15:30:00", "test_column_3": -128, "test_column_4": -128, "test_column_5": -128, "test_column_6": -128, "test_column_7": -128, "test_column_8": -128, "test_column_9": -128 }, "emitted_at": 1669643054045 } | ||
| {"stream": "sat_basic_dataset", "data": {"id": 2, "test_column_1": 127, "test_column_10": 127, "test_column_11": 127, "test_column_12": 127.0, "test_column_13": false, "test_column_15": "9999-12-31T00:00:00Z", "test_column_16": "9999-12-31T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 127, "test_column_9": 127 }, "emitted_at": 1669643054045 } | ||
| {"stream": "sat_basic_dataset", "data": {"id": 3, "test_column_1": 9223372036854775807, "test_column_10": 999999999999999999, "test_column_11": 999999999999999999, "test_column_12": 0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 999999999999999999, "test_column_9": 999999999999999999 }, "emitted_at": 1669643054045 } | ||
| {"stream": "sat_basic_dataset", "data": {"id": 7, "test_column_1": -9223372036854775808, "test_column_10": -0.123456789, "test_column_11": -0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -0.123456789, "test_column_9": -0.123456789 }, "emitted_at": 1669643054045 } | ||
| {"stream": "sat_basic_dataset", "data": {"id": 4, "test_column_1": -9223372036854775808, "test_column_10": -999999999999999999, "test_column_11": -999999999999999999, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -999999999999999999, "test_column_9": -999999999999999999 }, "emitted_at": 1669643054045 } | ||
| {"stream": "sat_basic_dataset", "data": {"id": 5, "test_column_1": -9223372036854775808, "test_column_10": 0.123456789, "test_column_11": 0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 0.123456789, "test_column_9": 0.123456789 }, "emitted_at": 1669643054045 } | ||
| {"stream": "sat_basic_dataset", "data": {"id": 6, "test_column_1": -9223372036854775808, "test_column_10": -0.123456789, "test_column_11": -0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -0.123456789, "test_column_9": -0.123456789 }, "emitted_at": 1669643054045 } |
47 changes: 47 additions & 0 deletions
47
airbyte-integrations/connectors/source-bigquery/integration_tests/full_dataset.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| CREATE TABLE | ||
| sat_test_dataset.sat_full_dataset(id NUMERIC(29), | ||
| test_column_1 int64, | ||
| test_column_10 decimal(29, | ||
| 9), | ||
| test_column_11 bigdecimal(76, | ||
| 38), | ||
| test_column_12 float64, | ||
| test_column_13 bool, | ||
| test_column_14 bytes, | ||
| test_column_15 date, | ||
| test_column_16 datetime, | ||
| test_column_17 timestamp, | ||
| test_column_18 geography, | ||
| test_column_19 string, | ||
| test_column_2 int, | ||
| test_column_20 STRUCT<course STRING, | ||
| id INT64>, | ||
| test_column_21 time, | ||
| test_column_22 ARRAY<String>, | ||
| test_column_23 STRUCT<frst String, | ||
| sec int64, | ||
| obbj STRUCT<id_col int64, | ||
| mega_obbj STRUCT<last_col time>>>, | ||
| test_column_24 ARRAY<STRUCT<fff String, | ||
| ggg int64>>, | ||
| test_column_25 ARRAY<STRUCT<fff String, | ||
| ggg ARRAY<STRUCT<ooo String, | ||
| kkk int64>>>>, | ||
| test_column_26 INTERVAL, | ||
| test_column_3 smallint, | ||
| test_column_4 integer, | ||
| test_column_5 bigint, | ||
| test_column_6 tinyint, | ||
| test_column_7 byteint, | ||
| test_column_8 numeric(29, | ||
| 9), | ||
| test_column_9 bignumeric(76, | ||
| 38) ); | ||
|
|
||
| INSERT INTO sat_test_dataset.sat_full_dataset VALUES (1, null, null, null, null, true, FROM_BASE64("test"), date('2021-10-20'), datetime('2021-10-20 11:22:33'), timestamp('2021-10-20 11:22:33'), ST_GEOGFROMTEXT('POINT(1 2)'), 'qwe', null, STRUCT("B.A",12), TIME(15, 30, 00), ['a', 'b'], STRUCT('s' as frst, 1 as sec, STRUCT(555 as id_col, STRUCT(TIME(15, 30, 00) as time) as mega_obbj) as obbj), [STRUCT('qqq' as fff, 1 as ggg), STRUCT('kkk' as fff, 2 as ggg)], [STRUCT('qqq' as fff, [STRUCT('fff' as ooo, 1 as kkk), STRUCT('hhh' as ooo, 2 as kkk)] as ggg)], MAKE_INTERVAL(2021, 10, 10, 10, 10, 10), null, null, null, null, null, null, null); | ||
| INSERT INTO sat_test_dataset.sat_full_dataset VALUES (2, -128, -128, -128, -128, false, null, date('9999-12-31'), datetime('9999-12-31 11:22:33'), null, null, 'йцу', -128, null, null, null, null, null, null, null, -128, -128, -128, -128, -128, -128, -128); | ||
| INSERT INTO sat_test_dataset.sat_full_dataset VALUES (3, 127, 127, 127, 127, null, null, date('0001-01-01'), datetime('0001-01-01 11:22:33'), null, null, null, 127, null, null, null, null, null, null, null, 127, 127, 127, 127, 127, 127, 127); | ||
| INSERT INTO sat_test_dataset.sat_full_dataset VALUES (4, 9223372036854775807, 999999999999999999, 999999999999999999, 0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 999999999999999999, 999999999999999999); | ||
| INSERT INTO sat_test_dataset.sat_full_dataset VALUES (5, -9223372036854775808, -999999999999999999, -999999999999999999, -0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, -999999999999999999, -999999999999999999); | ||
| INSERT INTO sat_test_dataset.sat_full_dataset VALUES (6, null, 0.123456789, 0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0.123456789, 0.123456789); | ||
| INSERT INTO sat_test_dataset.sat_full_dataset VALUES (7, null, -0.123456789, -0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, -0.123456789, -0.123456789); |