Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🎉 Source S3 - high strictness_level #21701

Merged
merged 1 commit into from
Jan 24, 2023
Merged

🎉 Source S3 - high strictness_level #21701

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 104 additions & 121 deletions airbyte-integrations/connectors/source-s3/acceptance-test-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,148 +1,131 @@
# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
# for more information about how to configure these tests
connector_image: airbyte/source-s3:dev
tests:
test_strictness_level: high
acceptance_tests:
spec:
- spec_path: "integration_tests/spec.json"
backward_compatibility_tests_config:
disable_for_version: "0.1.26"
tests:
- backward_compatibility_tests_config:
disable_for_version: 0.1.26
spec_path: integration_tests/spec.json

connection:
# for CSV format
- config_path: "secrets/config.json"
status: "succeed"
# for Parquet format
- config_path: "secrets/parquet_config.json"
status: "succeed"
# # for Avro format
- config_path: "secrets/avro_config.json"
status:
"succeed"
# for JSON format
- config_path: "secrets/jsonl_config.json"
status: "succeed"
- config_path: "secrets/jsonl_newlines_config.json"
status: "succeed"
# for custom server
- config_path: "integration_tests/config_minio.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
tests:
- config_path: secrets/config.json
status: succeed
- config_path: secrets/parquet_config.json
status: succeed
- config_path: secrets/avro_config.json
status: succeed
- config_path: secrets/jsonl_config.json
status: succeed
- config_path: secrets/jsonl_newlines_config.json
status: succeed
- config_path: integration_tests/config_minio.json
status: succeed
- config_path: integration_tests/invalid_config.json
status: failed

discovery:
# for CSV format
- config_path: "secrets/config.json"
# for Parquet format
- config_path: "secrets/parquet_config.json"
# for Avro format
- config_path: "secrets/avro_config.json"
# for JSON format
- config_path: "secrets/jsonl_config.json"
- config_path: "secrets/jsonl_newlines_config.json"
# for custom server
- config_path: "integration_tests/config_minio.json"
tests:
- config_path: secrets/config.json
- config_path: secrets/parquet_config.json
- config_path: secrets/avro_config.json
- config_path: secrets/jsonl_config.json
- config_path: secrets/jsonl_newlines_config.json
- config_path: integration_tests/config_minio.json

basic_read:
# for CSV format
- config_path: "secrets/config.json"
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/csv.json"
tests:
- config_path: secrets/config.json
expect_records:
path: "integration_tests/expected_records/csv.jsonl"
# for Parquet format
- config_path: "secrets/parquet_config.json"
path: integration_tests/expected_records/csv.jsonl
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/parquet.json"
- config_path: secrets/parquet_config.json
expect_records:
path: "integration_tests/expected_records/parquet.jsonl"
# for Avro format
- config_path: "secrets/avro_config.json"
path: integration_tests/expected_records/parquet.jsonl
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/avro.json"
- config_path: secrets/avro_config.json
expect_records:
path: "integration_tests/expected_records/avro.jsonl"
# for JSONL format
- config_path: "secrets/jsonl_config.json"
path: integration_tests/expected_records/avro.jsonl
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/jsonl.json"
- config_path: secrets/jsonl_config.json
expect_records:
path: "integration_tests/expected_records/jsonl.jsonl"
- config_path: "secrets/jsonl_newlines_config.json"
path: integration_tests/expected_records/jsonl.jsonl
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/jsonl.json"
- config_path: secrets/jsonl_newlines_config.json
expect_records:
path: "integration_tests/expected_records/jsonl_newlines.jsonl"
# for custom server
- config_path: "integration_tests/config_minio.json"
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/csv.json"
# expected records contains _ab_source_file_last_modified property which
# is modified all the time s3 file changed and for custom server it is
# file creating date and it always new. Uncomment this line when SAT
# would have ability to ignore specific fields from expected records.
# expect_records:
# path: "integration_tests/expected_records/custom_server.jsonl"
incremental:
# for CSV format
- config_path: "secrets/config.json"
path: integration_tests/expected_records/jsonl_newlines.jsonl
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/csv.json"
cursor_paths:
test: ["_ab_source_file_last_modified"]
future_state_path: "integration_tests/abnormal_state.json"
# for Parquet format
- config_path: "secrets/parquet_config.json"
# - config_path: integration_tests/config_minio.json
# timeout_seconds: 1800

full_refresh:
tests:
- config_path: secrets/config.json
configured_catalog_path: integration_tests/configured_catalogs/csv.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/parquet.json"
cursor_paths:
test: ["_ab_source_file_last_modified"]
future_state_path: "integration_tests/abnormal_state.json"
# for Avro format
- config_path: "secrets/avro_config.json"
- config_path: secrets/parquet_config.json
configured_catalog_path: integration_tests/configured_catalogs/parquet.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/avro.json"
cursor_paths:
test: ["_ab_source_file_last_modified"]
future_state_path: "integration_tests/abnormal_state.json"
# for JSON format
- config_path: "secrets/jsonl_config.json"
- config_path: secrets/avro_config.json
configured_catalog_path: integration_tests/configured_catalogs/avro.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/jsonl.json"
cursor_paths:
test: ["_ab_source_file_last_modified"]
future_state_path: "integration_tests/abnormal_state.json"
- config_path: "secrets/jsonl_newlines_config.json"
- config_path: secrets/jsonl_config.json
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/jsonl.json"
cursor_paths:
test: ["_ab_source_file_last_modified"]
future_state_path: "integration_tests/abnormal_state.json"
# for custom server
- config_path: "integration_tests/config_minio.json"
- config_path: secrets/jsonl_newlines_config.json
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
timeout_seconds: 1800
- config_path: integration_tests/config_minio.json
configured_catalog_path: integration_tests/configured_catalogs/csv.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/csv.json"
cursor_paths:
test: ["_ab_source_file_last_modified"]
future_state_path: "integration_tests/abnormal_state.json"

full_refresh:
# for CSV format
- config_path: "secrets/config.json"
incremental:
tests:
- config_path: secrets/config.json
configured_catalog_path: integration_tests/configured_catalogs/csv.json
cursor_paths:
test:
- _ab_source_file_last_modified
future_state:
future_state_path: integration_tests/abnormal_state.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/csv.json"
# for Parquet format
- config_path: "secrets/parquet_config.json"
- config_path: secrets/parquet_config.json
configured_catalog_path: integration_tests/configured_catalogs/parquet.json
cursor_paths:
test:
- _ab_source_file_last_modified
future_state:
future_state_path: integration_tests/abnormal_state.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/parquet.json"
# for Avro format
- config_path: "secrets/avro_config.json"
- config_path: secrets/avro_config.json
configured_catalog_path: integration_tests/configured_catalogs/avro.json
cursor_paths:
test:
- _ab_source_file_last_modified
future_state:
future_state_path: integration_tests/abnormal_state.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/avro.json"
# for JSON format
- config_path: "secrets/jsonl_config.json"
- config_path: secrets/jsonl_config.json
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
cursor_paths:
test:
- _ab_source_file_last_modified
future_state:
future_state_path: integration_tests/abnormal_state.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/jsonl.json"
- config_path: "secrets/jsonl_newlines_config.json"
- config_path: secrets/jsonl_newlines_config.json
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
cursor_paths:
test:
- _ab_source_file_last_modified
future_state:
future_state_path: integration_tests/abnormal_state.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/jsonl.json"
# for custom server
- config_path: "integration_tests/config_minio.json"
- config_path: integration_tests/config_minio.json
configured_catalog_path: integration_tests/configured_catalogs/csv.json
cursor_paths:
test:
- _ab_source_file_last_modified
future_state:
future_state_path: integration_tests/abnormal_state.json
timeout_seconds: 1800
configured_catalog_path: "integration_tests/configured_catalogs/csv.json"