diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 6b4f48c6c082..ce3c484e39f5 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.1.104 + +- Protocol change: `supported_sync_modes` is now a required properties on AirbyteStream. [#15591](https://github.com/airbytehq/airbyte/pull/15591) + ## 0.1.103 - Low-code: added hash filter to jinja template diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 8ccf522e6cdd..44c9c552c4a1 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -209,7 +209,7 @@ class Config: name: str = Field(..., description="Stream's name.") json_schema: Dict[str, Any] = Field(..., description="Stream schema using Json Schema specs.") - supported_sync_modes: Optional[List[SyncMode]] = None + supported_sync_modes: List[SyncMode] = Field(..., description="List of sync modes supported by this stream.", min_items=1) source_defined_cursor: Optional[bool] = Field( None, description="If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.", diff --git a/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py index fdfe9deb5d54..d5c6be46964a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py @@ -126,7 +126,7 @@ def singer_catalog_to_airbyte_catalog( for stream in singer_catalog.get("streams"): # type: ignore name = stream.get("stream") schema = stream.get("schema") - airbyte_stream = AirbyteStream(name=name, json_schema=schema) + airbyte_stream = AirbyteStream(name=name, json_schema=schema, supported_sync_modes=[SyncMode.full_refresh]) if name in sync_mode_overrides: override_sync_modes(airbyte_stream, sync_mode_overrides[name]) else: diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 9363d3ae5598..b509896a01f5 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.103", + version="0.1.104", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/destinations/test_destination.py b/airbyte-cdk/python/unit_tests/destinations/test_destination.py index 66043b0365bb..a0cd9b35f6ec 100644 --- a/airbyte-cdk/python/unit_tests/destinations/test_destination.py +++ b/airbyte-cdk/python/unit_tests/destinations/test_destination.py @@ -192,7 +192,7 @@ def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch dummy_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream(name="mystream", json_schema={"type": "object"}), + stream=AirbyteStream(name="mystream", json_schema={"type": "object"}, supported_sync_modes=[SyncMode.full_refresh]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-cdk/python/unit_tests/singer/test_singer_source.py b/airbyte-cdk/python/unit_tests/singer/test_singer_source.py index 58e53500595f..71156aad88ab 100644 --- a/airbyte-cdk/python/unit_tests/singer/test_singer_source.py +++ b/airbyte-cdk/python/unit_tests/singer/test_singer_source.py @@ -95,7 +95,7 @@ def test_singer_discover_metadata(mock_read_catalog): _user_stream = airbyte_catalog.streams[0] _roles_stream = airbyte_catalog.streams[1] - assert _user_stream.supported_sync_modes is None + # assert _user_stream.supported_sync_modes is None assert _user_stream.default_cursor_field is None assert _roles_stream.supported_sync_modes == [SyncMode.incremental] assert _roles_stream.default_cursor_field == ["name"] diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index a266b2756b8a..c81b794e5af6 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -60,12 +60,12 @@ def catalog(): configured_catalog = { "streams": [ { - "stream": {"name": "mock_http_stream", "json_schema": {}}, + "stream": {"name": "mock_http_stream", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh", }, { - "stream": {"name": "mock_stream", "json_schema": {}}, + "stream": {"name": "mock_stream", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh", }, @@ -317,7 +317,11 @@ def test_read_catalog(source): configured_catalog = { "streams": [ { - "stream": {"name": "mystream", "json_schema": {"type": "object", "properties": {"k": "v"}}}, + "stream": { + "name": "mystream", + "json_schema": {"type": "object", "properties": {"k": "v"}}, + "supported_sync_modes": ["full_refresh"], + }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh", } diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index 26bc5b855410..a71cd40eadd4 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -19,6 +19,7 @@ AirbyteStream, ConnectorSpecification, Status, + SyncMode, Type, ) from airbyte_cdk.sources import Source @@ -172,7 +173,7 @@ def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="discover", config="config_path") - expected = AirbyteCatalog(streams=[AirbyteStream(name="stream", json_schema={"k": "v"})]) + expected = AirbyteCatalog(streams=[AirbyteStream(name="stream", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh])]) mocker.patch.object(MockSource, "discover", return_value=expected) assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args)) assert spec_mock.called diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1d61824fdfd2..840676d25dad 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -96,7 +96,7 @@ - name: Azure Table Storage sourceDefinitionId: 798ae795-5189-42b6-b64e-3cb91db93338 dockerRepository: airbyte/source-azure-table - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.com/integrations/sources/azure-table icon: azureblobstorage.svg sourceType: database @@ -263,7 +263,7 @@ - name: E2E Testing sourceDefinitionId: d53f9084-fa6b-4a5a-976c-5b8392f4ad8a dockerRepository: airbyte/source-e2e-test - dockerImageTag: 2.1.1 + dockerImageTag: 2.1.3 documentationUrl: https://docs.airbyte.com/integrations/sources/e2e-test icon: airbyte.svg sourceType: api @@ -418,7 +418,7 @@ - name: Google Sheets sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 dockerRepository: airbyte/source-google-sheets - dockerImageTag: 0.2.20 + dockerImageTag: 0.2.21 documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets icon: google-sheets.svg sourceType: file diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5650e0811966..29b0af1dc01f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1319,7 +1319,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-azure-table:0.1.2" +- dockerImage: "airbyte/source-azure-table:0.1.3" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -1329,7 +1329,6 @@ required: - "storage_account_name" - "storage_access_key" - additionalProperties: false properties: storage_account_name: title: "Account Name" @@ -2410,7 +2409,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-e2e-test:2.1.1" +- dockerImage: "airbyte/source-e2e-test:2.1.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/e2e-test" connectionSpecification: @@ -2523,6 +2522,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + protocol_version: "0.2.1" - dockerImage: "airbyte/source-exchange-rates:1.2.6" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/exchangeratesapi" @@ -4353,7 +4353,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-sheets:0.2.20" +- dockerImage: "airbyte/source-google-sheets:0.2.21" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-sheets" connectionSpecification: diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelperTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelperTest.java index d5a4fce7d923..aaa903b29a57 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelperTest.java +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelperTest.java @@ -6,7 +6,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import com.google.common.collect.Lists; import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.SyncMode; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -17,32 +19,37 @@ class S3OutputPathHelperTest { public void testGetOutputPrefix() { // No namespace assertEquals("bucket_path/stream_name", S3OutputPathHelper - .getOutputPrefix("bucket_path", new AirbyteStream().withName("stream_name"))); + .getOutputPrefix("bucket_path", + new AirbyteStream().withName("stream_name").withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))); // With namespace assertEquals("bucket_path/namespace/stream_name", S3OutputPathHelper .getOutputPrefix("bucket_path", - new AirbyteStream().withNamespace("namespace").withName("stream_name"))); + new AirbyteStream().withNamespace("namespace").withName("stream_name") + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))); // With empty namespace assertEquals("bucket_path/stream_name", S3OutputPathHelper .getOutputPrefix("bucket_path", - new AirbyteStream().withNamespace("").withName("stream_name"))); + new AirbyteStream().withNamespace("").withName("stream_name").withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))); // With namespace with slash chart in the end assertEquals("bucket_path/namespace/stream_name", S3OutputPathHelper .getOutputPrefix("bucket_path", - new AirbyteStream().withNamespace("namespace/").withName("stream_name"))); + new AirbyteStream().withNamespace("namespace/").withName("stream_name") + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))); // With namespace with slash chart in the name assertEquals("bucket_path/namespace/subfolder/stream_name", S3OutputPathHelper .getOutputPrefix("bucket_path", - new AirbyteStream().withNamespace("namespace/subfolder/").withName("stream_name"))); + new AirbyteStream().withNamespace("namespace/subfolder/").withName("stream_name") + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))); // With an AWS Glue crawler assertEquals("bucket_path/namespace/date=2022-03-15", S3OutputPathHelper .getOutputPrefix("bucket_path", - new AirbyteStream().withNamespace("namespace").withName("date=2022-03-15"))); + new AirbyteStream().withNamespace("namespace").withName("date=2022-03-15") + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))); } } diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index 71fc49d543dc..7ec5868185dc 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.2.9 +Update tests after protocol change making `supported_sync_modes` a required property of `AirbyteStream` [#15591](https://github.com/airbytehq/airbyte/pull/15591/) + ## 0.2.8 Make full refresh tests tolerant to new records in a sequential read.[#17660](https://github.com/airbytehq/airbyte/pull/17660/) diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index 187f2a5c9138..516a33b4995b 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.2.8 +LABEL io.airbyte.version=0.2.9 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/setup.py b/airbyte-integrations/bases/source-acceptance-test/setup.py index 2fc14c1f7bbd..6bbbf11a2972 100644 --- a/airbyte-integrations/bases/source-acceptance-test/setup.py +++ b/airbyte-integrations/bases/source-acceptance-test/setup.py @@ -6,7 +6,7 @@ import setuptools MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.56", + "airbyte-cdk~=0.1.104", "docker~=5.0.3", "PyYAML~=5.4", "icdiff~=1.9", diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_asserts.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_asserts.py index 413eb608a95f..46d495abbbb0 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_asserts.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_asserts.py @@ -31,7 +31,7 @@ def record_schema_fixture(): def catalog_fixture(request, record_schema) -> ConfiguredAirbyteCatalog: record_schema = request.param if hasattr(request, "param") else record_schema stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="my_stream", json_schema=record_schema), + stream=AirbyteStream(name="my_stream", json_schema=record_schema, supported_sync_modes=[SyncMode.full_refresh]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.append, ) diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py index 011687ea44ed..f5105434b79c 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py @@ -999,12 +999,14 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ), "other_test_stream": AirbyteStream.parse_obj( { "name": "other_test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ), }, @@ -1013,6 +1015,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1025,6 +1028,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1033,6 +1037,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "integer"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1045,6 +1050,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1053,6 +1059,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "new_test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1065,6 +1072,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], "default_cursor_field": ["a"], } ), @@ -1074,6 +1082,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], "default_cursor_field": ["b"], } ), @@ -1087,6 +1096,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], "default_cursor_field": ["a"], } ), @@ -1096,6 +1106,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], "default_cursor_field": ["a", "b"], } ), @@ -1109,6 +1120,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], "default_cursor_field": ["a", "b"], } ), @@ -1118,6 +1130,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], "default_cursor_field": ["a"], } ), @@ -1131,6 +1144,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["incremental"], "default_cursor_field": ["a"], } ), @@ -1140,6 +1154,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["incremental"], "default_cursor_field": ["b"], } ), @@ -1147,6 +1162,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "other_test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["incremental"], } ), }, @@ -1160,6 +1176,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "integer"}}}}}, "default_cursor_field": ["a"], + "supported_sync_modes": ["incremental"], } ), }, @@ -1171,6 +1188,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "properties": {"user": {"type": "object", "properties": {"username": {"type": ["integer", "string"]}}}} }, "default_cursor_field": ["b"], + "supported_sync_modes": ["incremental"], } ), }, @@ -1186,6 +1204,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1194,12 +1213,14 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ), "other_test_stream": AirbyteStream.parse_obj( { "name": "other_test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ), }, @@ -1212,6 +1233,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1219,6 +1241,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", + "supported_sync_modes": ["full_refresh"], "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string", "null"]}}}}}, } ) @@ -1231,6 +1254,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", + "supported_sync_modes": ["full_refresh"], "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, } ) @@ -1240,6 +1264,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string"]}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1256,6 +1281,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "user": {"type": "object", "properties": {"username": {"type": "string"}, "email": {"type": "string"}}} } }, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1264,6 +1290,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe { "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1277,6 +1304,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, "default_cursor_field": ["a"], + "supported_sync_modes": ["full_refresh"], } ), }, @@ -1286,6 +1314,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, "default_cursor_field": ["a"], + "supported_sync_modes": ["full_refresh"], } ), }, diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py index 56b39d899564..6db6125eb4c8 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from unittest import mock from unittest.mock import MagicMock, patch import pytest @@ -15,6 +16,7 @@ ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Level, + SyncMode, TraceType, Type, ) @@ -39,7 +41,14 @@ def test_discovery(schema, cursors, should_fail): t = _TestDiscovery() discovered_catalog = { - "test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": schema, "default_cursor_field": cursors}) + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": schema, + "default_cursor_field": cursors, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ) } if should_fail: with pytest.raises(AssertionError): @@ -71,7 +80,11 @@ def test_discovery(schema, cursors, should_fail): ) def test_ref_in_discovery_schemas(schema, should_fail): t = _TestDiscovery() - discovered_catalog = {"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": schema})} + discovered_catalog = { + "test_stream": AirbyteStream.parse_obj( + {"name": "test_stream", "json_schema": schema, "supported_sync_modes": ["full_refresh", "incremental"]} + ) + } if should_fail: with pytest.raises(AssertionError): t.test_defined_refs_exist_in_schema(discovered_catalog) @@ -111,7 +124,11 @@ def test_ref_in_discovery_schemas(schema, should_fail): ) def test_keyword_in_discovery_schemas(schema, keyword, should_fail): t = _TestDiscovery() - discovered_catalog = {"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": schema})} + discovered_catalog = { + "test_stream": AirbyteStream.parse_obj( + {"name": "test_stream", "json_schema": schema, "supported_sync_modes": ["full_refresh", "incremental"]} + ) + } if should_fail: with pytest.raises(AssertionError): t.test_defined_keyword_exist_in_schema(keyword, discovered_catalog) @@ -122,30 +139,30 @@ def test_keyword_in_discovery_schemas(schema, keyword, should_fail): @pytest.mark.parametrize( "discovered_catalog, expectation", [ - ({"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": {}})}, pytest.raises(AssertionError)), + ({"test_stream": mock.MagicMock(name="test_stream", json_schema={}, supported_sync_modes=None)}, pytest.raises(AssertionError)), ( - {"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": {}, "supported_sync_modes": []})}, + {"test_stream": mock.MagicMock(name="test_stream", json_schema={}, supported_sync_modes=[])}, pytest.raises(AssertionError), ), ( { - "test_stream": AirbyteStream.parse_obj( - {"name": "test_stream", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"]} + "test_stream": mock.MagicMock( + name="test_stream", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] ) }, does_not_raise(), ), ( - {"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": {}, "supported_sync_modes": ["full_refresh"]})}, + {"test_stream": mock.MagicMock(name="test_stream", json_schema={}, supported_sync_modes=[SyncMode.full_refresh])}, does_not_raise(), ), ( - {"test_stream": AirbyteStream.parse_obj({"name": "test_stream", "json_schema": {}, "supported_sync_modes": ["incremental"]})}, + {"test_stream": mock.MagicMock(name="test_stream", json_schema={}, supported_sync_modes=[SyncMode.incremental])}, does_not_raise(), ), ], ) -def test_supported_sync_modes_in_stream(discovered_catalog, expectation): +def test_supported_sync_modes_in_stream(mocker, discovered_catalog, expectation): t = _TestDiscovery() with expectation: t.test_streams_has_sync_modes(discovered_catalog) @@ -154,17 +171,36 @@ def test_supported_sync_modes_in_stream(discovered_catalog, expectation): @pytest.mark.parametrize( "discovered_catalog, expectation", [ - ({"test_stream_1": AirbyteStream.parse_obj({"name": "test_stream_1", "json_schema": {}})}, does_not_raise()), ( - {"test_stream_2": AirbyteStream.parse_obj({"name": "test_stream_2", "json_schema": {"additionalProperties": True}})}, + { + "test_stream_1": AirbyteStream.parse_obj( + {"name": "test_stream_1", "json_schema": {}, "supported_sync_modes": ["full_refresh"]} + ) + }, does_not_raise(), ), ( - {"test_stream_3": AirbyteStream.parse_obj({"name": "test_stream_3", "json_schema": {"additionalProperties": False}})}, + { + "test_stream_2": AirbyteStream.parse_obj( + {"name": "test_stream_2", "json_schema": {"additionalProperties": True}, "supported_sync_modes": ["full_refresh"]} + ) + }, + does_not_raise(), + ), + ( + { + "test_stream_3": AirbyteStream.parse_obj( + {"name": "test_stream_3", "json_schema": {"additionalProperties": False}, "supported_sync_modes": ["full_refresh"]} + ) + }, pytest.raises(AssertionError), ), ( - {"test_stream_4": AirbyteStream.parse_obj({"name": "test_stream_4", "json_schema": {"additionalProperties": "foo"}})}, + { + "test_stream_4": AirbyteStream.parse_obj( + {"name": "test_stream_4", "json_schema": {"additionalProperties": "foo"}, "supported_sync_modes": ["full_refresh"]} + ) + }, pytest.raises(AssertionError), ), ( @@ -173,6 +209,7 @@ def test_supported_sync_modes_in_stream(discovered_catalog, expectation): { "name": "test_stream_5", "json_schema": {"additionalProperties": True, "properties": {"my_object": {"additionalProperties": True}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -184,6 +221,7 @@ def test_supported_sync_modes_in_stream(discovered_catalog, expectation): { "name": "test_stream_6", "json_schema": {"additionalProperties": True, "properties": {"my_object": {"additionalProperties": False}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -217,7 +255,7 @@ def test_read(schema, record, should_fail): catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream.parse_obj({"name": "test_stream", "json_schema": schema}), + stream=AirbyteStream.parse_obj({"name": "test_stream", "json_schema": schema, "supported_sync_modes": ["full_refresh"]}), sync_mode="full_refresh", destination_sync_mode="overwrite", ) @@ -334,7 +372,11 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur streams=[ ConfiguredAirbyteStream( stream=AirbyteStream.parse_obj( - {"name": "test1", "json_schema": {"type": "object", "properties": {"f1": {"type": "string"}}}} + { + "name": "test1", + "json_schema": {"type": "object", "properties": {"f1": {"type": "string"}}}, + "supported_sync_modes": ["full_refresh"], + } ), sync_mode="full_refresh", destination_sync_mode="overwrite", @@ -352,6 +394,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur { "name": "test1", "json_schema": {"type": "object", "properties": {"f1": {"type": "string"}, "f2": {"type": "string"}}}, + "supported_sync_modes": ["full_refresh"], } ), sync_mode="full_refresh", @@ -373,6 +416,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur { "name": "test1", "json_schema": {"type": "object", "properties": {"f1": {"type": "string"}, "f2": {"type": "string"}}}, + "supported_sync_modes": ["full_refresh"], } ), sync_mode="full_refresh", @@ -401,6 +445,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur "f3": {"type": "array", "items": {"type": "integer"}}, }, }, + "supported_sync_modes": ["full_refresh"], } ), sync_mode="full_refresh", @@ -429,6 +474,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur "f3": {"type": "array", "items": {"type": "integer"}}, }, }, + "supported_sync_modes": ["full_refresh"], } ), sync_mode="full_refresh", @@ -457,6 +503,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur "f3": {"type": "object", "properties": {"f4": {"type": "string"}, "f5": {"type": "array"}}}, }, }, + "supported_sync_modes": ["full_refresh"], } ), sync_mode="full_refresh", @@ -485,6 +532,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur "f3": {"type": "object", "properties": {"f4": {"type": "string"}, "f5": {"type": "array"}}}, }, }, + "supported_sync_modes": ["full_refresh"], } ), sync_mode="full_refresh", @@ -505,6 +553,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test1", + "supported_sync_modes": ["full_refresh"], "json_schema": { "type": "object", "properties": { @@ -552,6 +601,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test1", + "supported_sync_modes": ["full_refresh"], "json_schema": { "type": "object", "properties": { @@ -584,6 +634,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test2", + "supported_sync_modes": ["full_refresh"], "json_schema": {"type": "object", "properties": {"f8": {"type": "string"}, "f9": {"type": "string"}}}, } ), @@ -607,6 +658,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test1", + "supported_sync_modes": ["full_refresh"], "json_schema": { "type": "object", "properties": { @@ -639,6 +691,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test2", + "supported_sync_modes": ["full_refresh"], "json_schema": {"type": "object", "properties": {"f8": {"type": "string"}, "f9": {"type": "string"}}}, } ), @@ -661,6 +714,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test1", + "supported_sync_modes": ["full_refresh"], "json_schema": { "type": "object", "properties": { @@ -695,6 +749,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test1", + "supported_sync_modes": ["full_refresh"], "json_schema": { "type": "object", "properties": { @@ -739,6 +794,7 @@ def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failur stream=AirbyteStream.parse_obj( { "name": "test1", + "supported_sync_modes": ["full_refresh"], "json_schema": { "type": "object", "properties": { diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py index 67043769f140..e4e789e174c8 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_json_schema_helper.py @@ -64,7 +64,7 @@ def stream_schema_fixture(): def stream_mapping_fixture(stream_schema): return { "my_stream": ConfiguredAirbyteStream( - stream=AirbyteStream(name="my_stream", json_schema=stream_schema), + stream=AirbyteStream(name="my_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.full_refresh]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.append, ) diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_test_full_refresh.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_test_full_refresh.py index 54821dff87a3..24c6b7ff38c9 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_test_full_refresh.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_test_full_refresh.py @@ -7,7 +7,15 @@ import pytest from _pytest.outcomes import Failed -from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Type +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + SyncMode, + Type, +) from source_acceptance_test.config import ConnectionTestConfig from source_acceptance_test.tests.test_full_refresh import TestFullRefresh as _TestFullRefresh @@ -37,7 +45,7 @@ def get_default_catalog(schema, **kwargs): stream=AirbyteStream( name="test_stream", json_schema=schema, - supported_sync_modes=["full_refresh"], + supported_sync_modes=[SyncMode.full_refresh], ), **configured_catalog_kwargs, ) diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py index cb9d6401ed8d..db237b32eeb4 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py @@ -22,13 +22,13 @@ def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="append_stream", json_schema=stream_schema), + stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema), + stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestDataUtils.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestDataUtils.java index 7f1c473c68dc..f501cbf9531d 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestDataUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestDataUtils.java @@ -156,7 +156,8 @@ private static JsonNode getTestDataFromResourceJson(final String fileName) { public static ConfiguredAirbyteCatalog getCommonCatalog(final JsonNode schema, final String datasetId) { return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList(new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(USERS_STREAM_NAME).withNamespace(datasetId).withJsonSchema(schema)) + .withStream(new AirbyteStream().withName(USERS_STREAM_NAME).withNamespace(datasetId).withJsonSchema(schema) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withSyncMode(SyncMode.FULL_REFRESH).withDestinationSyncMode(DestinationSyncMode.OVERWRITE))); } diff --git a/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java b/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java index af9405a317c6..a0b469672476 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -13,6 +14,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; import java.time.Instant; import java.util.List; @@ -61,7 +63,8 @@ static AirbyteMessage createAirbyteMessage(AirbyteMessage.Type type, static AirbyteStream createAirbyteStream(String name, String namespace) { return new AirbyteStream() .withName(name) - .withNamespace(namespace); + .withNamespace(namespace) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)); } static ConfiguredAirbyteStream createConfiguredAirbyteStream(DestinationSyncMode syncMode, AirbyteStream stream) { diff --git a/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java b/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java index af9405a317c6..a0b469672476 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/TestDataFactory.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -13,6 +14,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; import java.time.Instant; import java.util.List; @@ -61,7 +63,8 @@ static AirbyteMessage createAirbyteMessage(AirbyteMessage.Type type, static AirbyteStream createAirbyteStream(String name, String namespace) { return new AirbyteStream() .withName(name) - .withNamespace(namespace); + .withNamespace(namespace) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)); } static ConfiguredAirbyteStream createConfiguredAirbyteStream(DestinationSyncMode syncMode, AirbyteStream stream) { diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationTest.java b/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationTest.java index 6b43938707ae..25043993bd34 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationTest.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.*; import org.junit.jupiter.api.Test; @@ -21,9 +22,8 @@ void testGetOutputTableNameWithString() throws Exception { @Test void testGetOutputTableNameWithStream() throws Exception { - final var stream = new AirbyteStream(); - stream.setName("test_stream"); - stream.setNamespace("test_namespace"); + final var stream = + new AirbyteStream().withName("test_stream").withNamespace("test_namespace").withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)); final var actual = DynamodbOutputTableHelper.getOutputTableName("test_table", stream); assertEquals("test_table_test_namespace_test_stream", actual); } diff --git a/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py index 9c4856855410..9d5e9e1c0c76 100644 --- a/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py @@ -60,7 +60,7 @@ def table_schema() -> str: @fixture def configured_catalogue(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name=test_table_name, json_schema=table_schema), + stream=AirbyteStream(name=test_table_name, json_schema=table_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) diff --git a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py index 8525c6114a02..64372a78f662 100644 --- a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py +++ b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py @@ -77,6 +77,7 @@ def configured_stream1() -> ConfiguredAirbyteStream: "type": "object", "properties": {"col1": {"type": "string"}, "col2": {"type": "integer"}}, }, + supported_sync_modes=[SyncMode.incremental], ), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, @@ -92,6 +93,7 @@ def configured_stream2() -> ConfiguredAirbyteStream: "type": "object", "properties": {"col1": {"type": "string"}, "col2": {"type": "integer"}}, }, + supported_sync_modes=[SyncMode.incremental], ), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, diff --git a/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py index 14987e41c5de..4602abb28e7a 100644 --- a/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py @@ -35,13 +35,13 @@ def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="append_stream", json_schema=stream_schema), + stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema), + stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java index c27da81fb152..05d5d27e697b 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java @@ -10,11 +10,13 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.SyncMode; import java.io.IOException; import java.sql.Timestamp; import java.time.Instant; @@ -35,7 +37,7 @@ public void generatesCorrectObjectPath() throws IOException { new ConfiguredAirbyteStream() .withStream(new AirbyteStream() .withNamespace("fake-namespace") - .withName("fake-stream")), + .withName("fake-stream").withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))), Timestamp.from(Instant.ofEpochMilli(1234)), null); diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java index a0f487d59018..acd777b32fda 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java @@ -14,6 +14,7 @@ import static org.mockito.Mockito.verify; import com.amazonaws.services.s3.AmazonS3Client; +import com.google.common.collect.Lists; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.destination.jdbc.SqlOperations; @@ -26,6 +27,7 @@ import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; import java.sql.Timestamp; import java.time.Instant; import java.util.ArrayList; @@ -54,7 +56,8 @@ public class S3StreamCopierTest { .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(new AirbyteStream() .withName("fake-stream") - .withNamespace("fake-namespace")); + .withNamespace("fake-namespace") + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))); private static final int UPLOAD_THREADS = 10; private static final int QUEUE_CAPACITY = 10; // equivalent to Thu, 09 Dec 2021 19:17:54 GMT diff --git a/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py index 73eba0b56631..8078180cb03d 100644 --- a/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py @@ -34,13 +34,13 @@ def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="append_stream", json_schema=stream_schema), + stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema), + stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py index 305817736c1e..db4958fc85f6 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py @@ -25,7 +25,7 @@ def _configured_catalog() -> ConfiguredAirbyteCatalog: stream_schema = {"type": "object", "properties": {"name": {"type": "string"}}} append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name=TEST_STREAM, json_schema=stream_schema), + stream=AirbyteStream(name=TEST_STREAM, json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) diff --git a/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py index 59ae8d924bab..39c4341ee2bf 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py @@ -78,7 +78,7 @@ def _record(stream: str, data: Dict[str, Any]) -> AirbyteMessage: def _configured_catalog() -> ConfiguredAirbyteCatalog: stream_schema = {"type": "object", "properties": {"name": {"type": "string"}, "email": {"type": "string"}}} append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="people", json_schema=stream_schema), + stream=AirbyteStream(name="people", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java index 6681540b0425..820897e0f598 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java @@ -24,6 +24,7 @@ import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; import java.sql.SQLException; import java.sql.Timestamp; import java.time.Instant; @@ -86,6 +87,7 @@ public void setup() { .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(new AirbyteStream() .withName("fake-stream") + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)) .withNamespace("fake-namespace"))); } diff --git a/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py index 165bb8233805..2401b99cefed 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py @@ -59,13 +59,13 @@ def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: } append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="append_stream", json_schema=stream_schema), + stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema), + stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierTest.java index a899e7562d60..4d34b1bebef2 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierTest.java @@ -19,6 +19,7 @@ import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; import java.sql.Timestamp; import java.time.Instant; import java.util.ArrayList; @@ -65,6 +66,7 @@ public void setup() throws Exception { .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(new AirbyteStream() .withName("fake-stream") + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)) .withNamespace("fake-namespace"))); } diff --git a/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py index 74ddb3466bb6..fcfba28b2b53 100644 --- a/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py @@ -58,7 +58,7 @@ def table_schema() -> str: @pytest.fixture def configured_catalogue(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name=test_table_name, json_schema=table_schema), + stream=AirbyteStream(name=test_table_name, json_schema=table_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) diff --git a/airbyte-integrations/connectors/source-azure-table/Dockerfile b/airbyte-integrations/connectors/source-azure-table/Dockerfile index e23b8cfe7db7..6b59fb5dbbb7 100644 --- a/airbyte-integrations/connectors/source-azure-table/Dockerfile +++ b/airbyte-integrations/connectors/source-azure-table/Dockerfile @@ -34,5 +34,5 @@ COPY source_azure_table ./source_azure_table ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-azure-table diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py b/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py index 1bd0c54cc740..d728ad4dd79b 100644 --- a/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/source.py @@ -2,7 +2,6 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import copy from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple from airbyte_cdk.logger import AirbyteLogger @@ -10,13 +9,13 @@ AirbyteCatalog, AirbyteConnectionStatus, AirbyteMessage, - AirbyteStateMessage, AirbyteStream, ConfiguredAirbyteCatalog, Status, - Type, + SyncMode, ) from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import split_config from airbyte_cdk.utils.event_timing import create_timer @@ -34,14 +33,6 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def _as_airbyte_record(self, stream_name: str, data: Mapping[str, Any]): return data - def _checkpoint_state(self, stream, stream_state, connector_state): - try: - connector_state[stream.name] = stream.state - except AttributeError: - connector_state[stream.name] = stream_state - - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=connector_state)) - @property def get_typed_schema(self) -> object: """Static schema for tables""" @@ -71,10 +62,13 @@ def discover(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteC streams = [] for table in tables: stream_name = table.name - stream = AirbyteStream(name=stream_name, json_schema=self.get_typed_schema) - stream.supported_sync_modes = ["full_refresh", "incremental"] - stream.source_defined_cursor = True - stream.default_cursor_field = ["PartitionKey"] + stream = AirbyteStream( + name=stream_name, + json_schema=self.get_typed_schema, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + source_defined_cursor=True, + default_cursor_field=["PartitionKey"], + ) streams.append(stream) logger.info(f"Total {streams.count} streams found.") @@ -106,10 +100,10 @@ def read( """ This method is overridden to check whether the stream `quotes` exists in the source, if not skip reading that stream. """ - connector_state = copy.deepcopy(state or {}) + stream_instances = {s.name: s for s in self.streams(logger=logger, config=config)} + state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state) logger.info(f"Starting syncing {self.name}") config, internal_config = split_config(config) - stream_instances = {s.name: s for s in self.streams(logger=logger, config=config)} self._stream_to_instance_map = stream_instances with create_timer(self.name) as timer: for configured_stream in catalog.streams: @@ -128,7 +122,7 @@ def read( logger=logger, stream_instance=stream_instance, configured_stream=configured_stream, - connector_state=connector_state, + state_manager=state_manager, internal_config=internal_config, ) except Exception as e: diff --git a/airbyte-integrations/connectors/source-azure-table/source_azure_table/spec.json b/airbyte-integrations/connectors/source-azure-table/source_azure_table/spec.json index b163fa470114..c314e5eea6c2 100644 --- a/airbyte-integrations/connectors/source-azure-table/source_azure_table/spec.json +++ b/airbyte-integrations/connectors/source-azure-table/source_azure_table/spec.json @@ -5,7 +5,6 @@ "title": "Azure Data Table Spec", "type": "object", "required": ["storage_account_name", "storage_access_key"], - "additionalProperties": false, "properties": { "storage_account_name": { "title": "Account Name", diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py index 2fe472fb8f05..8df88a5c196b 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py @@ -6,7 +6,7 @@ from unittest import mock import pytest -from airbyte_cdk.models import AirbyteCatalog +from airbyte_cdk.models import AirbyteCatalog, SyncMode from source_azure_table.source import SourceAzureTable from source_azure_table.streams import AzureTableStream @@ -49,7 +49,7 @@ def test_discover(mocker, config, tables): "type": "object", "properties": {"PartitionKey": {"type": "string"}}, } - assert stream.supported_sync_modes == ["full_refresh", "incremental"] + assert stream.supported_sync_modes == [SyncMode.full_refresh, SyncMode.incremental] assert stream.source_defined_cursor is True assert stream.default_cursor_field == ["PartitionKey"] diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile index b38a53fab502..10bf69c85f64 100644 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.1.2 +LABEL io.airbyte.version=2.1.3 LABEL io.airbyte.name=airbyte/source-e2e-test diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java index 57b69f8ace10..c23fbeac61b6 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedConfig.java @@ -5,12 +5,14 @@ package io.airbyte.integrations.source.e2e_test; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.SyncMode; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -79,13 +81,15 @@ static AirbyteCatalog parseMockCatalog(final JsonNode config) throws JsonValidat checkSchema(streamName, streamSchema.get()); if (streamDuplication == 1) { - final AirbyteStream stream = new AirbyteStream().withName(streamName).withJsonSchema(streamSchema.get()); + final AirbyteStream stream = new AirbyteStream().withName(streamName).withJsonSchema(streamSchema.get()) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)); return new AirbyteCatalog().withStreams(Collections.singletonList(stream)); } else { final List streams = new ArrayList<>(streamDuplication); for (int i = 0; i < streamDuplication; ++i) { streams.add(new AirbyteStream() .withName(String.join("_", streamName, String.valueOf(i))) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)) .withJsonSchema(streamSchema.get())); } return new AirbyteCatalog().withStreams(streams); @@ -104,7 +108,8 @@ static AirbyteCatalog parseMockCatalog(final JsonNode config) throws JsonValidat final String streamName = entry.getKey(); final JsonNode streamSchema = entry.getValue(); checkSchema(streamName, streamSchema); - streams.add(new AirbyteStream().withName(streamName).withJsonSchema(streamSchema)); + streams.add(new AirbyteStream().withName(streamName).withJsonSchema(streamSchema) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))); } return new AirbyteCatalog().withStreams(streams); } diff --git a/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json b/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json index 92f3f961c5a7..4fd00b2a75db 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json +++ b/airbyte-integrations/connectors/source-e2e-test/src/test/resources/parse_mock_catalog_test_cases.json @@ -20,7 +20,8 @@ "type": "number" } } - } + }, + "supported_sync_modes": ["full_refresh"] } ] } @@ -47,7 +48,8 @@ "type": "number" } } - } + }, + "supported_sync_modes": ["full_refresh"] }, { "name": "my_stream_1", @@ -61,7 +63,8 @@ "type": "number" } } - } + }, + "supported_sync_modes": ["full_refresh"] } ] } @@ -104,7 +107,8 @@ "type": "number" } } - } + }, + "supported_sync_modes": ["full_refresh"] }, { "name": "stream2", @@ -115,7 +119,8 @@ "type": "string" } } - } + }, + "supported_sync_modes": ["full_refresh"] } ] } diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 83bdc7857668..eeea91de04c0 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.20 +LABEL io.airbyte.version=0.2.21 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py index b22c4aa1521f..c321c416b8f9 100644 --- a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py @@ -10,7 +10,7 @@ from typing import Dict, FrozenSet, Iterable, List from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode from google.oauth2 import credentials as client_account from google.oauth2 import service_account from googleapiclient import discovery @@ -60,7 +60,7 @@ def headers_to_airbyte_stream(logger: AirbyteLogger, sheet_name: str, header_row "properties": {field: {"type": "string"} for field in fields}, } - return AirbyteStream(name=sheet_name, json_schema=sheet_json_schema, supported_sync_modes=["full_refresh"]) + return AirbyteStream(name=sheet_name, json_schema=sheet_json_schema, supported_sync_modes=[SyncMode.full_refresh]) @staticmethod def get_valid_headers_and_duplicates(header_row_values: List[str]) -> (List[str], List[str]): diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py index 227ecfb5b5b9..decb7fe26fba 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py @@ -35,7 +35,7 @@ def test_headers_to_airbyte_stream(self): # For simplicity, the type of every cell is a string "properties": {header: {"type": "string"} for header in header_values}, }, - supported_sync_modes=["full_refresh"], + supported_sync_modes=[SyncMode.full_refresh], ) actual_stream = Helpers.headers_to_airbyte_stream(logger, sheet_name, header_values) @@ -66,7 +66,7 @@ def test_duplicate_headers_to_ab_stream_ignores_duplicates(self): # For simplicity, the type of every cell is a string "properties": {header: {"type": "string"} for header in expected_stream_header_values}, }, - supported_sync_modes=["full_refresh"], + supported_sync_modes=[SyncMode.full_refresh], ) actual_stream = Helpers.headers_to_airbyte_stream(logger, sheet_name, header_values) @@ -84,7 +84,7 @@ def test_headers_to_airbyte_stream_blank_values_terminate_row(self): # For simplicity, the type of every cell is a string "properties": {"h1": {"type": "string"}}, }, - supported_sync_modes=["full_refresh"], + supported_sync_modes=[SyncMode.full_refresh], ) actual_stream = Helpers.headers_to_airbyte_stream(logger, sheet_name, header_values) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py index a97387026e33..b2fe9ecbcd40 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py @@ -65,7 +65,7 @@ def test_read(config): catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream(name="users", json_schema={}), + stream=AirbyteStream(name="users", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java index 7484202848b9..55df035c283d 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; import io.airbyte.integrations.source.relationaldb.models.DbState; @@ -29,6 +30,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.protocol.models.SyncMode; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -66,10 +68,12 @@ void testGetters() { final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() .withStreams(List.of( new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD1)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))))); final StateManager stateManager = new StreamStateManager(state, catalog); @@ -89,13 +93,16 @@ void testToState() { final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() .withStreams(List.of( new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD1)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD2)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))))); final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); @@ -151,13 +158,16 @@ void testToStateWithoutCursorInfo() { final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() .withStreams(List.of( new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD1)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD2)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))))); final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair = new AirbyteStreamNameNamespacePair("other", "other"); final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); @@ -172,13 +182,16 @@ void testToStateWithoutStreamPair() { final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() .withStreams(List.of( new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD1)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD2)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))))); final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); final AirbyteStateMessage airbyteStateMessage = stateManager.toState(Optional.empty()); @@ -193,10 +206,12 @@ void testToStateNullCursorField() { final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() .withStreams(List.of( new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withCursorField(List.of(CURSOR_FIELD1)), new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))))); final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); final DbState expectedFirstDbState = new DbState() diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index 9a816236ee6d..de776f3b129c 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -50,7 +50,8 @@ public static AirbyteStream createAirbyteStream(final String streamName, final S } public static AirbyteStream createAirbyteStream(final String streamName, final String namespace, final List fields) { - return new AirbyteStream().withName(streamName).withNamespace(namespace).withJsonSchema(fieldsToJsonSchema(fields)); + return new AirbyteStream().withName(streamName).withNamespace(namespace).withJsonSchema(fieldsToJsonSchema(fields)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)); } public static ConfiguredAirbyteCatalog createConfiguredAirbyteCatalog(final String streamName, final String namespace, final Field... fields) { @@ -67,7 +68,8 @@ public static ConfiguredAirbyteStream createConfiguredAirbyteStream(final String public static ConfiguredAirbyteStream createConfiguredAirbyteStream(final String streamName, final String namespace, final List fields) { return new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(streamName).withNamespace(namespace).withJsonSchema(fieldsToJsonSchema(fields))) + .withStream(new AirbyteStream().withName(streamName).withNamespace(namespace).withJsonSchema(fieldsToJsonSchema(fields)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) .withSyncMode(SyncMode.FULL_REFRESH).withDestinationSyncMode(DestinationSyncMode.OVERWRITE); } diff --git a/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 92a5c0924eb8..731ab00a332d 100644 --- a/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -228,8 +228,7 @@ definitions: required: - name - json_schema - # todo (cgardens) - make required once sources are migrated - # - supported_sync_modes + - supported_sync_modes properties: name: type: string @@ -239,7 +238,9 @@ definitions: type: object existingJavaType: com.fasterxml.jackson.databind.JsonNode supported_sync_modes: + description: List of sync modes supported by this stream. type: array + minItems: 1 items: "$ref": "#/definitions/SyncMode" source_defined_cursor: diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java index cc6c260848cd..761ef8f63a83 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java @@ -126,7 +126,7 @@ public class AirbyteAcceptanceTestHarness { private static final DockerImageName SOURCE_POSTGRES_IMAGE_NAME = DockerImageName.parse("debezium/postgres:13-alpine") .asCompatibleSubstituteFor("postgres"); - private static final String SOURCE_E2E_TEST_CONNECTOR_VERSION = "0.1.1"; + private static final String SOURCE_E2E_TEST_CONNECTOR_VERSION = "0.1.2"; private static final String DESTINATION_E2E_TEST_CONNECTOR_VERSION = "0.1.1"; public static final String POSTGRES_SOURCE_LEGACY_CONNECTOR_VERSION = "0.4.26"; diff --git a/docs/integrations/sources/azure-table.md b/docs/integrations/sources/azure-table.md index 78c3bc42f7cd..108b90361c94 100644 --- a/docs/integrations/sources/azure-table.md +++ b/docs/integrations/sources/azure-table.md @@ -33,13 +33,13 @@ Azure Table Storage uses different [property](https://docs.microsoft.com/en-us/r ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Incremental - Dedupe Sync | No | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :------------------------ | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Incremental - Dedupe Sync | No | +| SSL connection | Yes | +| Namespaces | No | ### Performance considerations @@ -65,8 +65,9 @@ We recommend creating a restricted key specifically for Airbyte access. This wil ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.2 | 2021-12-23 | [14212](https://github.com/airbytehq/airbyte/pull/14212) | Adding incremental load capability | -| 0.1.1 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.3 | 2022-08-12 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Clean instantiation of AirbyteStream | +| 0.1.2 | 2021-12-23 | [14212](https://github.com/airbytehq/airbyte/pull/14212) | Adding incremental load capability | +| 0.1.1 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index ab2a0a8f185c..12a9a7ce22df 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -16,15 +16,15 @@ The single-stream catalog config exists just for convenient, since in many testi Here is its configuration: -| Mock Catalog Type | Parameters | Type | Required | Default | Notes | -| --- | --- | --- | --- | --- | --- | -| Single-stream | stream name | string | yes | | Name of the stream in the catalog. | -| | stream schema | json | yes | | Json schema of the stream in the catalog. It must be a valid Json schema. | -| | stream duplication | integer | no | 1 | Duplicate the stream N times to quickly create a multi-stream catalog. | -| Multi-stream | streams and schemas | json | yes | | A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. | -| Both | max records | integer | yes | 100 | The number of record messages to emit from this connector. Min 1. Max 100 billion. | -| | random seed | integer | no | current time millis | The seed is used in random Json object generation. Min 0. Max 1 million. | -| | message interval | integer | no | 0 | The time interval between messages in millisecond. Min 0 ms. Max 60000 ms (1 minute). | +| Mock Catalog Type | Parameters | Type | Required | Default | Notes | +| ----------------- | ------------------- | ------- | -------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Single-stream | stream name | string | yes | | Name of the stream in the catalog. | +| | stream schema | json | yes | | Json schema of the stream in the catalog. It must be a valid Json schema. | +| | stream duplication | integer | no | 1 | Duplicate the stream N times to quickly create a multi-stream catalog. | +| Multi-stream | streams and schemas | json | yes | | A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. | +| Both | max records | integer | yes | 100 | The number of record messages to emit from this connector. Min 1. Max 100 billion. | +| | random seed | integer | no | current time millis | The seed is used in random Json object generation. Min 0. Max 1 million. | +| | message interval | integer | no | 0 | The time interval between messages in millisecond. Min 0 ms. Max 60000 ms (1 minute). | ### Legacy Infinite Feed @@ -46,10 +46,10 @@ This mode can generate infinite number of records, which can be dangerous. That' There are two configurable parameters: -| Parameters | Type | Required | Default | Notes | -| --- | --- | --- | --- | --- | -| max records | integer | no | `null` | Number of message records to emit. When it is left empty, the connector will generate infinite number of messages. | -| message interval | integer | no | `null` | Time interval between messages in millisecond. | +| Parameters | Type | Required | Default | Notes | +| ---------------- | ------- | -------- | ------- | ------------------------------------------------------------------------------------------------------------------ | +| max records | integer | no | `null` | Number of message records to emit. When it is left empty, the connector will generate infinite number of messages. | +| message interval | integer | no | `null` | Time interval between messages in millisecond. | ### Exception after N @@ -61,13 +61,14 @@ This mode is also excluded from the Cloud variant of this connector. The OSS and Cloud variants have the same version number. The Cloud variant was initially released at version `1.0.0`. -| Version | Date | Pull request | Notes | -| --- | --- | --- | --- | -| 2.1.1 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 2.1.0 | 2021-02-12 | [\#10298](https://github.com/airbytehq/airbyte/pull/10298) | Support stream duplication to quickly create a multi-stream catalog. | -| 2.0.0 | 2021-02-01 | [\#9954](https://github.com/airbytehq/airbyte/pull/9954) | Remove legacy modes. Use more efficient Json generator. | -| 1.0.1 | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | -| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | -| 0.1.2 | 2022-10-18 | [\#18100](https://github.com/airbytehq/airbyte/pull/18100) | Set supported sync mode on streams | -| 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | -| 0.1.0 | 2021-07-23 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) [\#4939](https://github.com/airbytehq/airbyte/pull/4939) | Initial release. | +| Version | Date | Pull request | Notes | +| ------- | ---------- | ----------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| 2.1.3 | 2022-08-25 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Declare supported sync modes in catalogs | +| 2.1.1 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 2.1.0 | 2021-02-12 | [\#10298](https://github.com/airbytehq/airbyte/pull/10298) | Support stream duplication to quickly create a multi-stream catalog. | +| 2.0.0 | 2021-02-01 | [\#9954](https://github.com/airbytehq/airbyte/pull/9954) | Remove legacy modes. Use more efficient Json generator. | +| 1.0.1 | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | +| 0.1.2 | 2022-10-18 | [\#18100](https://github.com/airbytehq/airbyte/pull/18100) | Set supported sync mode on streams | +| 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | +| 0.1.0 | 2021-07-23 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) [\#4939](https://github.com/airbytehq/airbyte/pull/4939) | Initial release. | diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index f78f741a8f62..976be4c54037 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -71,6 +71,7 @@ The [Google API rate limit](https://developers.google.com/sheets/api/limits) is | Version | Date | Pull Request | Subject | | ------- | ---------- | -------------------------------------------------------- | ----------------------------------------------------------------------------- | +| 0.2.21 | 2022-10-04 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Clean instantiation of AirbyteStream | | 0.2.20 | 2022-10-10 | [17766](https://github.com/airbytehq/airbyte/pull/17766) | Fix null pointer exception when parsing the spreadsheet id. | | 0.2.19 | 2022-09-29 | [17410](https://github.com/airbytehq/airbyte/pull/17410) | Use latest CDK. | | 0.2.18 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. |