diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9a95011bf289..a33867fcf3d3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1730,7 +1730,7 @@ - name: S3 sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 dockerRepository: airbyte/source-s3 - dockerImageTag: 2.0.0 + dockerImageTag: 2.0.1 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 icon: s3.svg sourceType: file diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 345ada7dcf4e..c18fccd9a1d0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -12988,7 +12988,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-s3:2.0.0" +- dockerImage: "airbyte/source-s3:2.0.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/s3" changelogUrl: "https://docs.airbyte.com/integrations/sources/s3" diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile index efe74dd39b4e..5b6a42999e05 100644 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ b/airbyte-integrations/connectors/source-s3/Dockerfile @@ -17,5 +17,5 @@ COPY source_s3 ./source_s3 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.0 +LABEL io.airbyte.version=2.0.1 LABEL io.airbyte.name=airbyte/source-s3 diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json index 1a18a33307b3..3793b4eb4f32 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json @@ -3,7 +3,7 @@ "type": "STREAM", "stream": { "stream_state": { - "_ab_source_file_last_modified": "2999-01-01T00:00:00+0000" + "_ab_source_file_last_modified": "2999-01-01T00:00:00Z" }, "stream_descriptor": { "name": "test" } } diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json index fa13fb3b9ca0..6011d8c41149 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json @@ -6,7 +6,7 @@ "aws_access_key_id": "123456", "aws_secret_access_key": "123456key", "path_prefix": "", - "endpoint": "http://10.0.186.80:9000" + "endpoint": "http://10.0.186.241:9000" }, "format": { "filetype": "csv", diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl index 697501058c33..383fa6fc868d 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl @@ -1,10 +1,10 @@ -{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl index c0d6452ed123..a36e69c9579e 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl index ef52db86be11..95e621c6494e 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl @@ -1,55 +1,55 @@ -{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl index fc86bc1a3d06..529953af7cf2 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl @@ -1,2 +1,2 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T08:31:02+0000", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T08:31:02+0000", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T08:31:02Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T08:31:02Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl index d4d6e09f1663..bb9161a55c93 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl @@ -1,2 +1,2 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T10:07:00+0000", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T10:07:00+0000", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T10:07:00Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2022-07-15T10:07:00Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl index 986478dbee87..19360fd6b65f 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl @@ -1,4 +1,4 @@ -{"stream": "test", "data": {"number": 1.0, "name": "foo", "flag": true, "delta": -1.0, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17+0000", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} -{"stream": "test", "data": {"number": 2.0, "name": null, "flag": false, "delta": 2.5, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17+0000", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} -{"stream": "test", "data": {"number": 3.0, "name": "bar", "flag": null, "delta": 0.1, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17+0000", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} -{"stream": "test", "data": {"number": null, "name": "baz", "flag": true, "delta": null, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17+0000", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} +{"stream": "test", "data": {"number": 1.0, "name": "foo", "flag": true, "delta": -1.0, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} +{"stream": "test", "data": {"number": 2.0, "name": null, "flag": false, "delta": 2.5, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} +{"stream": "test", "data": {"number": 3.0, "name": "bar", "flag": null, "delta": 0.1, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} +{"stream": "test", "data": {"number": null, "name": "baz", "flag": true, "delta": null, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py b/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py index 1ae6079e1b5d..3d78cf03ae9c 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py @@ -104,7 +104,7 @@ def _stream_records_test_logic( # emulate state for incremental testing # since we're not actually saving state out to file here, we pass schema in to our FileStream creation... # this isn't how it will work in Airbyte but it's a close enough emulation - current_state = state if state is not None else {FileStream.ab_last_mod_col: "1970-01-01T00:00:00+0000"} + current_state = state if state is not None else {FileStream.ab_last_mod_col: "1970-01-01T00:00:00Z"} if (user_schema is None) and ("schema" in current_state.keys()): user_schema = current_state["schema"] diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/__init__.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py index 9283d2a652ac..170b7957b2da 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py @@ -11,6 +11,7 @@ from traceback import format_exc from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union +import pytz from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.streams import Stream @@ -49,7 +50,7 @@ def fileformatparser_map(self) -> Mapping[str, type]: ab_last_mod_col = "_ab_source_file_last_modified" ab_file_name_col = "_ab_source_file_url" airbyte_columns = [ab_additional_col, ab_last_mod_col, ab_file_name_col] - datetime_format_string = "%Y-%m-%dT%H:%M:%S%z" + datetime_format_string = "%Y-%m-%dT%H:%M:%SZ" def __init__(self, dataset: str, provider: dict, format: dict, path_pattern: str, schema: str = None): """ @@ -310,11 +311,17 @@ def file_in_history(file_info: FileInfo, history: dict) -> bool: return False def _get_datetime_from_stream_state(self, stream_state: Mapping[str, Any] = None) -> datetime: - """if no state, we default to 1970-01-01 in order to pick up all files present.""" + """ + Returns the datetime from the stream state. + + If there is no state, defaults to 1970-01-01 in order to pick up all files present. + The datetime object is localized to UTC to match the timezone of the last_modified attribute of objects in S3. + """ if stream_state is not None and self.cursor_field in stream_state.keys(): - return datetime.strptime(stream_state[self.cursor_field], self.datetime_format_string) + state_datetime = datetime.strptime(stream_state[self.cursor_field], self.datetime_format_string) else: - return datetime.strptime("1970-01-01T00:00:00+0000", self.datetime_format_string) + state_datetime = datetime.strptime("1970-01-01T00:00:00Z", self.datetime_format_string) + return pytz.utc.localize(state_datetime) def get_updated_history(self, current_stream_state, latest_record_datetime, latest_record, current_parsed_datetime, state_date): """ @@ -367,8 +374,8 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late """ state_dict: Dict[str, Any] = {} current_parsed_datetime = self._get_datetime_from_stream_state(current_stream_state) - latest_record_datetime = datetime.strptime( - latest_record.get(self.cursor_field, "1970-01-01T00:00:00+0000"), self.datetime_format_string + latest_record_datetime = pytz.utc.localize( + datetime.strptime(latest_record.get(self.cursor_field, "1970-01-01T00:00:00Z"), self.datetime_format_string) ) state_dict[self.cursor_field] = datetime.strftime(max(current_parsed_datetime, latest_record_datetime), self.datetime_format_string) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py index 0c025365f144..8dc35c5501df 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py @@ -340,23 +340,23 @@ def test_pattern_matched_filepath_iterator(self, patterns: str, filepaths: List[ "latest_record, current_stream_state, expected", [ ( # overwrite history file - {"id": 1, "_ab_source_file_last_modified": "2022-05-11T11:54:11+0000", "_ab_source_file_url": "new_test_file.csv"}, - {"_ab_source_file_last_modified": "2021-07-25T15:33:04+0000", "history": {"2021-07-25": {"old_test_file.csv"}}}, + {"id": 1, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "new_test_file.csv"}, + {"_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "history": {"2021-07-25": {"old_test_file.csv"}}}, {"2022-05-11": {"new_test_file.csv"}}, ), ( # add file to same day - {"id": 1, "_ab_source_file_last_modified": "2022-07-25T11:54:11+0000", "_ab_source_file_url": "new_test_file.csv"}, - {"_ab_source_file_last_modified": "2022-07-25T00:00:00+0000", "history": {"2022-07-25": {"old_test_file.csv"}}}, + {"id": 1, "_ab_source_file_last_modified": "2022-07-25T11:54:11Z", "_ab_source_file_url": "new_test_file.csv"}, + {"_ab_source_file_last_modified": "2022-07-25T00:00:00Z", "history": {"2022-07-25": {"old_test_file.csv"}}}, {"2022-07-25": {"new_test_file.csv", "old_test_file.csv"}}, ), ( # add new day to history - {"id": 1, "_ab_source_file_last_modified": "2022-07-03T11:54:11+0000", "_ab_source_file_url": "new_test_file.csv"}, - {"_ab_source_file_last_modified": "2022-07-01T00:00:00+0000", "history": {"2022-07-01": {"old_test_file.csv"}}}, + {"id": 1, "_ab_source_file_last_modified": "2022-07-03T11:54:11Z", "_ab_source_file_url": "new_test_file.csv"}, + {"_ab_source_file_last_modified": "2022-07-01T00:00:00Z", "history": {"2022-07-01": {"old_test_file.csv"}}}, {"2022-07-01": {"old_test_file.csv"}, "2022-07-03": {"new_test_file.csv"}}, ), ( # history size limit reached {"_ab_source_file_url": "test.csv"}, - {"_ab_source_file_last_modified": "2022-07-01T00:00:00+0000", "history": mock_big_size_object()}, + {"_ab_source_file_last_modified": "2022-07-01T00:00:00Z", "history": mock_big_size_object()}, None, ), ], @@ -377,7 +377,7 @@ def test_get_updated_history(self, latest_record, current_stream_state, expected "stream_state, expected_error", [ (None, False), - ({"_ab_source_file_last_modified": "2021-07-25T15:33:04+0000"}, False), + ({"_ab_source_file_last_modified": "2021-07-25T15:33:04Z"}, False), ({"_ab_source_file_last_modified": "2021-07-25T15:33:04Z"}, False), ({"_ab_source_file_last_modified": "2021-07-25"}, True), ], @@ -416,7 +416,7 @@ def test_read(self): stream_instance.read_records( stream_slice=slice, sync_mode=SyncMode.full_refresh, - stream_state={"_ab_source_file_last_modified": "1999-01-01T00:00:00+0000"}, + stream_state={"_ab_source_file_last_modified": "1999-01-01T00:00:00Z"}, ) ) ) diff --git a/connectors.md b/connectors.md index b820499dd141..ea8498dfc561 100644 --- a/connectors.md +++ b/connectors.md @@ -198,7 +198,7 @@ | **Reply.io** | Reply.io icon | Source | airbyte/source-reply-io:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/reply-io) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-reply-io) | `8cc6537e-f8a6-423c-b960-e927af76116e` | | **Retently** | Retently icon | Source | airbyte/source-retently:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/sources/retently) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-retently) | `db04ecd1-42e7-4115-9cec-95812905c626` | | **Rocket.chat** | Rocket.chat icon | Source | airbyte/source-rocket-chat:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/rocket-chat) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-rocket-chat) | `921d9608-3915-450b-8078-0af18801ea1b` | -| **S3** | S3 icon | Source | airbyte/source-s3:2.0.0 | generally_available | [link](https://docs.airbyte.com/integrations/sources/s3) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-s3) | `69589781-7828-43c5-9f63-8925b1c1ccc2` | +| **S3** | S3 icon | Source | airbyte/source-s3:2.0.1 | generally_available | [link](https://docs.airbyte.com/integrations/sources/s3) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-s3) | `69589781-7828-43c5-9f63-8925b1c1ccc2` | | **SAP Fieldglass** | SAP Fieldglass icon | Source | airbyte/source-sap-fieldglass:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/sap-fieldglass) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sap-fieldglass) | `ec5f3102-fb31-4916-99ae-864faf8e7e25` | | **SFTP** | SFTP icon | Source | airbyte/source-sftp:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/sources/sftp) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sftp) | `a827c52e-791c-4135-a245-e233c5255199` | | **SFTP Bulk** | SFTP Bulk icon | Source | airbyte/source-sftp-bulk:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/sftp-bulk) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sftp-bulk) | `31e3242f-dee7-4cdc-a4b8-8e06c5458517` | diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 837ca66fa7f5..e6d77a7874e7 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -215,6 +215,7 @@ The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is suppo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------| +| 2.0.1 | 2023-03-06 | [23195](https://github.com/airbytehq/airbyte/pull/23195) | Fix datetime format string | | 2.0.0 | 2023-03-14 | [23189](https://github.com/airbytehq/airbyte/pull/23189) | Infer schema based on one file instead of all the files | | 1.0.2 | 2023-03-02 | [23669](https://github.com/airbytehq/airbyte/pull/23669) | Made `Advanced Reader Options` and `Advanced Options` truly `optional` for `CSV` format | | 1.0.1 | 2023-02-27 | [23502](https://github.com/airbytehq/airbyte/pull/23502) | Fix error handling |