Source S3: Add a few more CAT tests (#29069)

* Add a few more CAT tests * rename * Add missing file
airbytehq · Aug 4, 2023 · 9f8ce9d · 9f8ce9d
1 parent 9dffd0c
commit 9f8ce9d
Show file tree

Hide file tree

Showing 13 changed files with 145 additions and 0 deletions.
diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml
@@ -5,6 +5,30 @@ acceptance_tests:
       expect_records:
         path: integration_tests/expected_records/csv.jsonl
       timeout_seconds: 1800
+    - config_path: secrets/legacy_csv_custom_encoding_config.json
+      expect_records:
+        path: integration_tests/expected_records/legacy_csv_custom_encoding.jsonl
+      timeout_seconds: 1800
+    - config_path: secrets/legacy_csv_custom_format_config.json
+      expect_records:
+        path: integration_tests/expected_records/legacy_csv_custom_format.jsonl
+      timeout_seconds: 1800
+    - config_path: secrets/legacy_csv_user_schema_config.json
+      expect_records:
+        path: integration_tests/expected_records/legacy_csv_user_schema.jsonl
+      timeout_seconds: 1800
+    - config_path: secrets/legacy_csv_no_header_config.json
+      expect_records:
+        path: integration_tests/expected_records/legacy_csv_no_header.jsonl
+      timeout_seconds: 1800
+    - config_path: secrets/legacy_csv_skip_rows_config.json
+      expect_records:
+        path: integration_tests/expected_records/legacy_csv_skip_rows.jsonl
+      timeout_seconds: 1800
+    - config_path: secrets/legacy_csv_with_nulls_config.json
+      expect_records:
+        path: integration_tests/expected_records/legacy_csv_with_nulls.jsonl
+      timeout_seconds: 1800
     - config_path: secrets/parquet_config.json
       expect_records:
         path: integration_tests/expected_records/parquet.jsonl
@@ -30,6 +54,18 @@ acceptance_tests:
     tests:
     - config_path: secrets/config.json
       status: succeed
+    - config_path: secrets/legacy_csv_custom_encoding_config.json
+      status: succeed
+    - config_path: secrets/legacy_csv_custom_format_config.json
+      status: succeed
+    - config_path: secrets/legacy_csv_user_schema_config.json
+      status: succeed
+    - config_path: secrets/legacy_csv_no_header_config.json
+      status: succeed
+    - config_path: secrets/legacy_csv_skip_rows_config.json
+      status: succeed
+    - config_path: secrets/legacy_csv_with_nulls_config.json
+      status: succeed
     - config_path: secrets/parquet_config.json
       status: succeed
     - config_path: secrets/avro_config.json
@@ -43,6 +79,12 @@ acceptance_tests:
   discovery:
     tests:
     - config_path: secrets/config.json
+    - config_path: secrets/legacy_csv_custom_encoding_config.json
+    - config_path: secrets/legacy_csv_custom_format_config.json
+    - config_path: secrets/legacy_csv_user_schema_config.json
+    - config_path: secrets/legacy_csv_no_header_config.json
+    - config_path: secrets/legacy_csv_skip_rows_config.json
+    - config_path: secrets/legacy_csv_with_nulls_config.json
     - config_path: secrets/parquet_config.json
     - config_path: secrets/avro_config.json
     - config_path: secrets/jsonl_config.json

diff --git a/.../connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl b/.../connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl
@@ -0,0 +1,8 @@
+{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1\u20ac", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 162727468000}
+{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
diff --git a/...ns/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl b/...ns/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl
@@ -0,0 +1,8 @@
+{"stream": "test", "data": {"id": 1, "name": "PVdhmj|b1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 162727468000}
+{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
diff --git a/...ations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl b/...ations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl
@@ -0,0 +1,8 @@
+{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 162727468000}
+{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
diff --git a/...ations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl b/...ations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl
@@ -0,0 +1,8 @@
+{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 162727468000}
+{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
diff --git a/...ions/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl b/...ions/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl
@@ -0,0 +1,8 @@
+{"stream": "test", "data": {"id": 1.0, "name": "PVdhmjb1", "valid": false, "valid_string": "False", "array": "[\"a\", \"b\", \"c\"]", "dict": "{\"key\": \"value\"}","_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 162727468000}
+{"stream": "test", "data": {"id": 2.0, "name": "j4DyXTS7", "valid": true, "valid_string": "True", "array": "[\"a\", \"b\"]","dict": "{\"key\": \"value_with_comma\\,\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 3.0, "name": "v0w8fTME", "valid": false, "valid_string": "False", "array": "[\"a\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 4.0, "name": "1q6jD8Np", "valid": false, "valid_string": "False", "array": "[]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 5.0, "name": "77h4aiMP", "valid": true, "valid_string": "True", "array": "[\"b\", \"c\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 6.0, "name": "Le35Wyic", "valid": true, "valid_string": "True", "array": "[\"a\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 7.0, "name": "xZhh1Kyl", "valid": false, "valid_string": "False", "array": "[\"b\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 8.0, "name": "M2t286iJ", "valid": false, "valid_string": "False", "array":  "[\"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
diff --git a/...tions/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl b/...tions/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl
@@ -0,0 +1,8 @@
+{"stream": "test", "data": {"id": 1, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 162727468000}
+{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 3, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
+{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
diff --git a/...ons/connectors/source-s3/integration_tests/sample_files/csvfile/csv_encoded_as_cp1252.csv b/...ons/connectors/source-s3/integration_tests/sample_files/csvfile/csv_encoded_as_cp1252.csv
@@ -0,0 +1,9 @@
+id,name,valid
+1,PVdhmjb1�,False
+2,j4DyXTS7,True
+3,v0w8fTME,False
+4,1q6jD8Np,False
+5,77h4aiMP,True
+6,Le35Wyic,True
+7,xZhh1Kyl,False
+8,M2t286iJ,False
diff --git a/...tegrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_with_nulls.csv b/...tegrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_with_nulls.csv
@@ -0,0 +1,9 @@
+id,name,valid
+1,NULL,False
+2,j4DyXTS7,True
+3,NONE,False
+4,1q6jD8Np,False
+5,77h4aiMP,True
+6,Le35Wyic,True
+7,xZhh1Kyl,False
+8,M2t286iJ,False
diff --git a/...ntegrations/connectors/source-s3/integration_tests/sample_files/csvfile/custom_format.csv b/...ntegrations/connectors/source-s3/integration_tests/sample_files/csvfile/custom_format.csv
@@ -0,0 +1,9 @@
+id|name|valid
+1|PVdhmj@|b1|False
+2|j4DyXTS7|True
+3|v0w8fTME|False
+4|1q6jD8Np|False
+5|77h4aiMP|True
+6|Le35Wyic|True
+7|xZhh1Kyl|False
+8|M2t286iJ|False
diff --git a/...te-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/datatypes.csv b/...te-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/datatypes.csv
@@ -0,0 +1,9 @@
+id,name,valid,valid_string,array,dict
+1,PVdhmjb1,False,False,"[""a"", ""b"", ""c""]","{""key"": ""value""}"
+2,j4DyXTS7,True,True,"[""a"", ""b""]","{""key"": ""value_with_comma\,""}"
+3,v0w8fTME,False,False,"[""a""]","{""key"": ""value""}"
+4,1q6jD8Np,False,False,"[]","{}"
+5,77h4aiMP,True,True,"[""b"", ""c""]","{}"
+6,Le35Wyic,True,True,"[""a"", ""c""]","{}"
+7,xZhh1Kyl,False,False,"[""b""]","{}"
+8,M2t286iJ,False,False,"[""c""]","{}"
diff --git a/...te-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/no_header.csv b/...te-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/no_header.csv
@@ -0,0 +1,8 @@
+1,PVdhmjb1,False
+2,j4DyXTS7,True
+3,v0w8fTME,False
+4,1q6jD8Np,False
+5,77h4aiMP,True
+6,Le35Wyic,True
+7,xZhh1Kyl,False
+8,M2t286iJ,False
diff --git a/...te-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/skip_rows.csv b/...te-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/skip_rows.csv
@@ -0,0 +1,11 @@
+skip_this_line
+id,name,valid
+skip,this,line
+1,PVdhmjb1,False
+2,j4DyXTS7,True
+3,v0w8fTME,False
+4,1q6jD8Np,False
+5,77h4aiMP,True
+6,Le35Wyic,True
+7,xZhh1Kyl,False
+8,M2t286iJ,False