From 2d60438cd88cdd5f3643079d47e60c10d374a41f Mon Sep 17 00:00:00 2001 From: Augustin Date: Fri, 5 Aug 2022 08:24:21 +0200 Subject: [PATCH 01/25] SAT: new tests for spec backward compatibility - syntactic validation (#15194) --- .../bases/source-acceptance-test/CHANGELOG.md | 3 + .../bases/source-acceptance-test/Dockerfile | 2 +- .../bases/source-acceptance-test/pytest.ini | 1 + .../bases/source-acceptance-test/setup.py | 1 + .../source_acceptance_test/config.py | 8 +- .../source_acceptance_test/conftest.py | 32 +- .../source_acceptance_test/plugin.py | 4 + .../source_acceptance_test/tests/test_core.py | 179 +++- .../unit_tests/test_backward_compatibility.py | 977 ++++++++++++++++++ .../unit_tests/test_spec.py | 38 - .../source-acceptance-tests-reference.md | 4 +- 11 files changed, 1155 insertions(+), 94 deletions(-) create mode 100644 airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index 43a344cac6df..4735703751e0 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.59 +Backward compatibility tests: add syntactic validation of specs [#15194](https://github.com/airbytehq/airbyte/pull/15194/). + ## 0.1.58 Bootstrap spec backward compatibility tests. Add fixtures to retrieve a previous connector version spec [#14954](https://github.com/airbytehq/airbyte/pull/14954/). diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index f1b78978db01..92d1f22eb9bb 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.1.58 +LABEL io.airbyte.version=0.1.59 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/pytest.ini b/airbyte-integrations/bases/source-acceptance-test/pytest.ini index e1827862065a..1eff47fef43c 100644 --- a/airbyte-integrations/bases/source-acceptance-test/pytest.ini +++ b/airbyte-integrations/bases/source-acceptance-test/pytest.ini @@ -6,3 +6,4 @@ testpaths = markers = default_timeout + backward_compatibility diff --git a/airbyte-integrations/bases/source-acceptance-test/setup.py b/airbyte-integrations/bases/source-acceptance-test/setup.py index dfffee08a770..6b53663573c2 100644 --- a/airbyte-integrations/bases/source-acceptance-test/setup.py +++ b/airbyte-integrations/bases/source-acceptance-test/setup.py @@ -20,6 +20,7 @@ "dpath~=2.0.1", "jsonschema~=3.2.0", "jsonref==0.2", + "deepdiff~=5.8.0", "requests-mock", "pytest-mock~=3.6.1", ] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py index b974d03e0b7f..dfcbc8ba33db 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py @@ -17,6 +17,8 @@ configured_catalog_path: Optional[str] = Field(default=None, description="Path to configured catalog") timeout_seconds: int = Field(default=None, description="Test execution timeout_seconds", ge=0) +SEMVER_REGEX = r"(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*)))?(?:\-([\w][\w\.\-_]*))?)?" + class BaseConfig(BaseModel): class Config: @@ -25,10 +27,10 @@ class Config: class BackwardCompatibilityTestsConfig(BaseConfig): previous_connector_version: str = Field( - default="latest", description="Previous connector version to use for backward compatibility tests." + regex=SEMVER_REGEX, default="latest", description="Previous connector version to use for backward compatibility tests." ) - disable_backward_compatibility_tests_for_version: Optional[str] = Field( - default=None, description="Disable backward compatibility tests for a specific connector version." + disable_for_version: Optional[str] = Field( + regex=SEMVER_REGEX, default=None, description="Disable backward compatibility tests for a specific connector version." ) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py index 6fc61ff76b70..301267c0329a 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py @@ -23,8 +23,9 @@ Type, ) from docker import errors +from source_acceptance_test.base import BaseTest from source_acceptance_test.config import Config -from source_acceptance_test.utils import ConnectorRunner, SecretDict, load_config, load_yaml_or_json_path +from source_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, load_config, load_yaml_or_json_path @pytest.fixture(name="base_path") @@ -210,6 +211,35 @@ def detailed_logger() -> Logger: return logger +@pytest.fixture(name="actual_connector_spec") +def actual_connector_spec_fixture(request: BaseTest, docker_runner: ConnectorRunner) -> ConnectorSpecification: + if not request.instance.spec_cache: + output = docker_runner.call_spec() + spec_messages = filter_output(output, Type.SPEC) + assert len(spec_messages) == 1, "Spec message should be emitted exactly once" + spec = spec_messages[0].spec + request.instance.spec_cache = spec + return request.instance.spec_cache + + +@pytest.fixture(name="previous_connector_spec") +def previous_connector_spec_fixture( + request: BaseTest, previous_connector_docker_runner: ConnectorRunner +) -> Optional[ConnectorSpecification]: + if previous_connector_docker_runner is None: + logging.warning( + "\n We could not retrieve the previous connector spec as a connector runner for the previous connector version could not be instantiated." + ) + return None + if not request.instance.previous_spec_cache: + output = previous_connector_docker_runner.call_spec() + spec_messages = filter_output(output, Type.SPEC) + assert len(spec_messages) == 1, "Spec message should be emitted exactly once" + spec = spec_messages[0].spec + request.instance.previous_spec_cache = spec + return request.instance.previous_spec_cache + + def pytest_sessionfinish(session, exitstatus): """Called after whole test run finished, right before returning the exit status to the system. https://docs.pytest.org/en/6.2.x/reference.html#pytest.hookspec.pytest_sessionfinish diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py index 3875cad4dd29..0145ce88699b 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py @@ -16,6 +16,10 @@ def pytest_configure(config): config.addinivalue_line("markers", "default_timeout: mark test to be wrapped by `timeout` decorator with default value") + config.addinivalue_line( + "markers", + "backward_compatibility: mark test to be part of the backward compatibility tests suite (deselect with '-m \"not backward_compatibility\"')", + ) def pytest_load_initial_conftests(early_config: Config, parser: Parser, args: List[str]): diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index d5986b53561a..068bd614e6c8 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -8,7 +8,7 @@ from collections import Counter, defaultdict from functools import reduce from logging import Logger -from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Set +from typing import Any, Dict, List, Mapping, MutableMapping, Set import dpath.util import jsonschema @@ -24,6 +24,7 @@ TraceType, Type, ) +from deepdiff import DeepDiff from docker.errors import ContainerError from jsonschema._utils import flatten from source_acceptance_test.base import BaseTest @@ -38,41 +39,38 @@ def connector_spec_dict_fixture(actual_connector_spec): return json.loads(actual_connector_spec.json()) -@pytest.fixture(name="actual_connector_spec") -def actual_connector_spec_fixture(request: BaseTest, docker_runner: ConnectorRunner) -> ConnectorSpecification: - if not request.instance.spec_cache: - output = docker_runner.call_spec() - spec_messages = filter_output(output, Type.SPEC) - assert len(spec_messages) == 1, "Spec message should be emitted exactly once" - spec = spec_messages[0].spec - request.spec_cache = spec - return request.spec_cache - - -@pytest.fixture(name="previous_connector_spec") -def previous_connector_spec_fixture( - request: BaseTest, previous_connector_docker_runner: ConnectorRunner -) -> Optional[ConnectorSpecification]: - if previous_connector_docker_runner is None: - logging.warning( - "\n We could not retrieve the previous connector spec as a connector runner for the previous connector version could not be instantiated." - ) - return None - if not request.instance.previous_spec_cache: - output = previous_connector_docker_runner.call_spec() - spec_messages = filter_output(output, Type.SPEC) - assert len(spec_messages) == 1, "Spec message should be emitted exactly once" - spec = spec_messages[0].spec - request.instance.previous_spec_cache = spec - return request.instance.previous_spec_cache - - @pytest.mark.default_timeout(10) class TestSpec(BaseTest): spec_cache: ConnectorSpecification = None previous_spec_cache: ConnectorSpecification = None + @staticmethod + def compute_spec_diff(actual_connector_spec: ConnectorSpecification, previous_connector_spec: ConnectorSpecification): + return DeepDiff(previous_connector_spec.dict(), actual_connector_spec.dict(), view="tree", ignore_order=True) + + @pytest.fixture(name="skip_backward_compatibility_tests") + def skip_backward_compatibility_tests_fixture(self, inputs: SpecTestConfig, previous_connector_docker_runner: ConnectorRunner) -> bool: + if previous_connector_docker_runner is None: + pytest.skip("The previous connector image could not be retrieved.") + + # Get the real connector version in case 'latest' is used in the config: + previous_connector_version = previous_connector_docker_runner._image.labels.get("io.airbyte.version") + + if previous_connector_version == inputs.backward_compatibility_tests_config.disable_for_version: + pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") + return False + + @pytest.fixture(name="spec_diff") + def spec_diff_fixture( + self, + skip_backward_compatibility_tests: bool, + actual_connector_spec: ConnectorSpecification, + previous_connector_spec: ConnectorSpecification, + ) -> DeepDiff: + assert isinstance(actual_connector_spec, ConnectorSpecification) and isinstance(previous_connector_spec, ConnectorSpecification) + return self.compute_spec_diff(actual_connector_spec, previous_connector_spec) + def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict): """Check that config matches the actual schema from the spec call""" # Getting rid of technical variables that start with an underscore @@ -182,7 +180,108 @@ def test_oauth_flow_parameters(self, actual_connector_spec: ConnectorSpecificati diff = params - schema_path assert diff == set(), f"Specified oauth fields are missed from spec schema: {diff}" - def test_additional_properties_is_true(self, actual_connector_spec): + @pytest.mark.default_timeout(60) + @pytest.mark.backward_compatibility + def test_new_required_field_declaration(self, spec_diff: DeepDiff): + """Check if a 'required' field was added to the spec.""" + added_required_fields = [ + addition for addition in spec_diff.get("dictionary_item_added", []) if addition.path(output_format="list")[-1] == "required" + ] + assert len(added_required_fields) == 0, f"The current spec has a new required field: {spec_diff.pretty()}" + + @pytest.mark.default_timeout(60) + @pytest.mark.backward_compatibility + def test_new_required_property(self, spec_diff: DeepDiff): + """Check if a a new property was added to the 'required' field.""" + added_required_properties = [ + addition for addition in spec_diff.get("iterable_item_added", []) if addition.up.path(output_format="list")[-1] == "required" + ] + assert len(added_required_properties) == 0, f"The current spec has a new required property: {spec_diff.pretty()}" + + @pytest.mark.default_timeout(60) + @pytest.mark.backward_compatibility + def test_field_type_changed(self, spec_diff: DeepDiff): + """Check if the current spec is changing the types of properties.""" + + common_error_message = f"The current spec changed the value of a 'type' field: {spec_diff.pretty()}" + # Detect type value change in case type field is declared as a string (e.g "str" -> "int"): + type_values_changed = [change for change in spec_diff.get("values_changed", []) if change.path(output_format="list")[-1] == "type"] + + # Detect type value change in case type field is declared as a single item list (e.g ["str"] -> ["int"]): + type_values_changed_in_list = [ + change for change in spec_diff.get("values_changed", []) if change.path(output_format="list")[-2] == "type" + ] + + assert len(type_values_changed_in_list) == 0 and len(type_values_changed) == 0, common_error_message + + # Detect type value added to type list if new type value is not None (e.g ["str"] -> ["str", "int"]): + # It works because we compute the diff with 'ignore_order=True' + new_values_in_type_list = [ # noqa: F841 + change + for change in spec_diff.get("iterable_item_added", []) + if change.path(output_format="list")[-2] == "type" + if change.t2 != "null" + ] + # enable the assertion below if we want to disallow type expansion: + # assert len(new_values_in_type_list) == 0, common_error_message + + # Detect the change of type of a type field + # e.g: + # - "str" -> ["str"] VALID + # - "str" -> ["str", "null"] VALID + # - "str" -> ["str", "int"] VALID + # - "str" -> 1 INVALID + # - ["str"] -> "str" VALID + # - ["str"] -> "int" INVALID + # - ["str"] -> 1 INVALID + type_changes = [change for change in spec_diff.get("type_changes", []) if change.path(output_format="list")[-1] == "type"] + for change in type_changes: + # We only accept change on the type field if the new type for this field is list or string + # This might be something already guaranteed by JSON schema validation. + if isinstance(change.t1, str): + assert isinstance( + change.t2, list + ), f"The current spec change a type field from string to an invalid value: {spec_diff.pretty()}" + assert ( + 0 < len(change.t2) <= 2 + ), f"The current spec change a type field from string to an invalid value. The type list length should not be empty and have a maximum of two items {spec_diff.pretty()}." + # If the new type field is a list we want to make sure it only has the original type (t1) and null: e.g. "str" -> ["str", "null"] + # We want to raise an error otherwise. + t2_not_null_types = [_type for _type in change.t2 if _type != "null"] + assert ( + len(t2_not_null_types) == 1 and t2_not_null_types[0] == change.t1 + ), "The current spec change a type field to a list with multiple invalid values." + if isinstance(change.t1, list): + assert isinstance( + change.t2, str + ), f"The current spec change a type field from list to an invalid value: {spec_diff.pretty()}" + assert len(change.t1) == 1 and change.t2 == change.t1[0], f"The current spec narrowed a field type: {spec_diff.pretty()}" + + # Detect when field was made not nullable but is still a list: e.g ["string", "null"] -> ["string"] + removed_nullable = [ + change for change in spec_diff.get("iterable_item_removed", []) if change.path(output_format="list")[-2] == "type" + ] + assert len(removed_nullable) == 0, f"The current spec narrowed a field type: {spec_diff.pretty()}" + + @pytest.mark.default_timeout(60) + @pytest.mark.backward_compatibility + def test_enum_field_has_narrowed(self, spec_diff: DeepDiff): + """Check if the list of values in a enum was shortened in a spec.""" + removals = [ + removal for removal in spec_diff.get("iterable_item_removed", []) if removal.up.path(output_format="list")[-1] == "enum" + ] + assert len(removals) == 0, f"The current spec narrowed a field enum: {spec_diff.pretty()}" + + @pytest.mark.default_timeout(60) + @pytest.mark.backward_compatibility + def test_new_enum_field_declaration(self, spec_diff: DeepDiff): + """Check if an 'enum' field was added to the spec.""" + added_enum_fields = [ + addition for addition in spec_diff.get("dictionary_item_added", []) if addition.path(output_format="list")[-1] == "enum" + ] + assert len(added_enum_fields) == 0, f"An 'enum' field was declared on an existing property of the spec: {spec_diff.pretty()}" + + def test_additional_properties_is_true(self, actual_connector_spec: ConnectorSpecification): """Check that value of the "additionalProperties" field is always true. A spec declaring "additionalProperties": false introduces the risk of accidental breaking changes. Specifically, when removing a property from the spec, existing connector configs will no longer be valid. @@ -196,24 +295,6 @@ def test_additional_properties_is_true(self, actual_connector_spec): [additional_properties_value is True for additional_properties_value in additional_properties_values] ), "When set, additionalProperties field value must be true for backward compatibility." - @pytest.mark.default_timeout(60) # Pulling the previous connector image can take more than 10 sec. - @pytest.mark.spec_backward_compatibility - def test_backward_compatibility( - self, inputs: SpecTestConfig, actual_connector_spec: ConnectorSpecification, previous_connector_spec: ConnectorSpecification - ): - """Run multiple checks to make sure the actual_connector_spec is backward compatible with the previous_connector_spec""" - if ( - inputs.backward_compatibility_tests_config.disable_backward_compatibility_tests_for_version - == inputs.backward_compatibility_tests_config.previous_connector_version - ): - pytest.skip( - f"Backward compatibility tests are disabled for version {inputs.backward_compatibility_tests_config.disable_backward_compatibility_tests_for_version}." - ) - if previous_connector_spec is None: - pytest.skip("The previous connector spec could not be retrieved.") - assert isinstance(actual_connector_spec, ConnectorSpecification) and isinstance(previous_connector_spec, ConnectorSpecification) - # TODO alafanechere: add the actual tests for backward compatibility below or in a dedicated module. - @pytest.mark.default_timeout(30) class TestConnection(BaseTest): diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py new file mode 100644 index 000000000000..e97ee150c247 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py @@ -0,0 +1,977 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.models import ConnectorSpecification +from source_acceptance_test.tests.test_core import TestSpec as _TestSpec + +from .conftest import does_not_raise + + +@pytest.mark.parametrize( + "connector_spec, expectation", + [ + (ConnectorSpecification(connectionSpecification={}), does_not_raise()), + (ConnectorSpecification(connectionSpecification={"type": "object", "additionalProperties": True}), does_not_raise()), + (ConnectorSpecification(connectionSpecification={"type": "object", "additionalProperties": False}), pytest.raises(AssertionError)), + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "additionalProperties": True, + "properties": {"my_object": {"type": "object", "additionalProperties": "foo"}}, + } + ), + pytest.raises(AssertionError), + ), + ( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "additionalProperties": True, + "properties": { + "my_oneOf_object": {"type": "object", "oneOf": [{"additionalProperties": True}, {"additionalProperties": False}]} + }, + } + ), + pytest.raises(AssertionError), + ), + ], +) +def test_additional_properties_is_true(connector_spec, expectation): + t = _TestSpec() + with expectation: + t.test_additional_properties_is_true(connector_spec) + + +@pytest.mark.parametrize( + "previous_connector_spec, actual_connector_spec, expectation", + [ + pytest.param( + ConnectorSpecification(connectionSpecification={}), + ConnectorSpecification( + connectionSpecification={ + "required": ["a", "b"], + } + ), + pytest.raises(AssertionError), + id="Top level: declaring the required field should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": {"my_optional_object": {"type": "object", "properties": {"optional_property": {"type": "string"}}}}, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_optional_object": { + "type": "object", + "required": ["optional_property"], + "properties": {"optional_property": {"type": "string"}}, + } + }, + } + ), + pytest.raises(AssertionError), + id="Nested level: adding the required field should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "required": ["my_required_string"], + "properties": { + "my_required_string": {"type": "string"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "required": ["my_required_string"], + "properties": { + "my_required_string": {"type": "string"}, + "my_optional_object": { + "type": "object", + "required": ["another_required_string"], + "properties": {"another_required_string": {"type": "string"}}, + }, + }, + } + ), + does_not_raise(), + id="Adding an optional object with required properties should not fail.", + ), + ], +) +def test_new_required_field_declaration(previous_connector_spec, actual_connector_spec, expectation): + t = _TestSpec() + spec_diff = t.compute_spec_diff(actual_connector_spec, previous_connector_spec) + with expectation: + t.test_new_required_field_declaration(spec_diff) + + +@pytest.mark.parametrize( + "previous_connector_spec, actual_connector_spec, expectation", + [ + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "required": ["a"], + } + ), + ConnectorSpecification( + connectionSpecification={ + "required": ["a", "b"], + } + ), + pytest.raises(AssertionError), + id="Top level: adding a new required property should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_optional_object": { + "type": "object", + "required": ["first_required_property"], + "properties": {"first_required_property": {"type": "string"}}, + } + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_optional_object": { + "type": "object", + "required": ["first_required_property", "second_required_property"], + "properties": { + "first_required_property": {"type": "string"}, + "second_required_property": {"type": "string"}, + }, + } + }, + } + ), + pytest.raises(AssertionError), + id="Nested level: adding a new required property should fail.", + ), + ], +) +def test_new_required_property(previous_connector_spec, actual_connector_spec, expectation): + t = _TestSpec() + spec_diff = t.compute_spec_diff(actual_connector_spec, previous_connector_spec) + with expectation: + t.test_new_required_property(spec_diff) + + +@pytest.mark.parametrize( + "previous_connector_spec, actual_connector_spec, expectation", + [ + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "int"}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + does_not_raise(), + id="No change should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + does_not_raise(), + id="Changing a field type from a string to a list should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["int"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type from a string to a list with a different type value should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "int"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["int", "int"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type from a string to a list with duplicate same type should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "int"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["int", "null", "str"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type from a string to a list with more than two values should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "int"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": []}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type from a string to an empty list should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["int"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": []}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type from a list to an empty list should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "int"}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type should fail from a list to string with different value should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + does_not_raise(), + id="Changing a field type from a list to a string with same value should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["int"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type in list should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str", "int"]}, + }, + } + ), + does_not_raise(), + id="Adding a field type in list should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["null", "str"]}, + }, + } + ), + does_not_raise(), + id="Making a field nullable should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str", "null"]}, + }, + } + ), + does_not_raise(), + id="Making a field nullable should not fail (change list order).", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["null", "str"]}, + }, + } + ), + does_not_raise(), + id="Making a field nullable should not fail (from a list).", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str", "null"]}, + }, + } + ), + does_not_raise(), + id="Making a field nullable should not fail (from a list, changing order).", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["int", "null"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Making a field nullable and changing type should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["null", "int"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Making a field nullable and changing type should fail (change list order).", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": "str"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": 1}, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type from a string to something else than a list should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["null", "str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["null", "int"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Nullable field: Changing a field type should fail", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["null", "str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str", "null"]}, + }, + } + ), + does_not_raise(), + id="Nullable field: Changing order should not fail", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["null", "str"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_int": {"type": ["str"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Nullable field: Making a field not nullable should fail", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": ["null", "string"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string"}, + }, + } + ), + pytest.raises(AssertionError), + id="Nullable: Making a field not nullable should fail (not in a list).", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": ["null", "int"]}}}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": ["int"]}}}, + }, + } + ), + pytest.raises(AssertionError), + id="Nested level: Narrowing a field type should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": ["int"]}}}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": ["null", "int"]}}}, + }, + } + ), + does_not_raise(), + id="Nested level: Expanding a field type should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "oneOf": [ + {"title": "a", "type": "str"}, + {"title": "b", "type": "int"}, + ] + }, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "oneOf": [ + {"title": "a", "type": "int"}, + {"title": "b", "type": "int"}, + ] + }, + }, + } + ), + pytest.raises(AssertionError), + id="Changing a field type in oneOf should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "oneOf": [ + {"title": "a", "type": "str"}, + {"title": "b", "type": "int"}, + ] + }, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "oneOf": [ + {"title": "b", "type": "str"}, + {"title": "a", "type": "int"}, + ] + }, + }, + } + ), + does_not_raise(), + id="Changing a order in oneOf should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "oneOf": [ + {"title": "a", "type": ["str", "int"]}, + {"title": "b", "type": "int"}, + ] + }, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "credentials": { + "oneOf": [ + {"title": "a", "type": ["str"]}, + {"title": "b", "type": "int"}, + ] + }, + }, + } + ), + pytest.raises(AssertionError), + id="Narrowing a field type in oneOf should fail.", + ), + ], +) +def test_field_type_changed(previous_connector_spec, actual_connector_spec, expectation): + t = _TestSpec() + spec_diff = t.compute_spec_diff(actual_connector_spec, previous_connector_spec) + with expectation: + t.test_field_type_changed(spec_diff) + + +@pytest.mark.parametrize( + "previous_connector_spec, actual_connector_spec, expectation", + [ + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "enum": ["a", "b"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "enum": ["a"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Top level: Narrowing a field enum should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "enum": ["a"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "enum": ["a", "b"]}, + }, + } + ), + does_not_raise(), + id="Top level: Expanding a field enum should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string", "enum": ["a", "b"]}}}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string", "enum": ["a"]}}}, + }, + } + ), + pytest.raises(AssertionError), + id="Nested level: Narrowing a field enum should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string", "enum": ["a"]}}}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string", "enum": ["a", "b"]}}}, + }, + } + ), + does_not_raise(), + id="Nested level: Expanding a field enum should not fail.", + ), + ], +) +def test_enum_field_has_narrowed(previous_connector_spec, actual_connector_spec, expectation): + t = _TestSpec() + spec_diff = t.compute_spec_diff(actual_connector_spec, previous_connector_spec) + with expectation: + t.test_enum_field_has_narrowed(spec_diff) + + +@pytest.mark.parametrize( + "previous_connector_spec, actual_connector_spec, expectation", + [ + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "enum": ["a", "b"]}, + }, + } + ), + pytest.raises(AssertionError), + id="Top level: Declaring a field enum should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "enum": ["a", "b"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string"}, + }, + } + ), + does_not_raise(), + id="Top level: Removing the field enum should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "enum": ["a", "b"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": {"my_string": {"type": "string", "enum": ["a", "b"]}, "my_enum": {"type": "string", "enum": ["c", "d"]}}, + } + ), + does_not_raise(), + id="Top level: Adding a new optional field with enum should not fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string"}}}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string", "enum": ["a", "b"]}}}, + }, + } + ), + pytest.raises(AssertionError), + id="Nested level: Declaring a field enum should fail.", + ), + pytest.param( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string", "enum": ["a", "b"]}}}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": "string"}}}, + }, + } + ), + does_not_raise(), + id="Nested level: Removing the enum field should not fail.", + ), + ], +) +def test_new_enum_field_declaration(previous_connector_spec, actual_connector_spec, expectation): + t = _TestSpec() + spec_diff = t.compute_spec_diff(actual_connector_spec, previous_connector_spec) + with expectation: + t.test_new_enum_field_declaration(spec_diff) diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_spec.py index 3a09ce07f887..ea8fe2d57dc9 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_spec.py @@ -8,8 +8,6 @@ from airbyte_cdk.models import ConnectorSpecification from source_acceptance_test.tests.test_core import TestSpec as _TestSpec -from .conftest import does_not_raise - @pytest.mark.parametrize( "connector_spec, should_fail", @@ -569,39 +567,3 @@ def test_validate_oauth_flow(connector_spec, expected_error): t.test_oauth_flow_parameters(connector_spec) else: t.test_oauth_flow_parameters(connector_spec) - - -@pytest.mark.parametrize( - "connector_spec, expectation", - [ - (ConnectorSpecification(connectionSpecification={}), does_not_raise()), - (ConnectorSpecification(connectionSpecification={"type": "object", "additionalProperties": True}), does_not_raise()), - (ConnectorSpecification(connectionSpecification={"type": "object", "additionalProperties": False}), pytest.raises(AssertionError)), - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "additionalProperties": True, - "properties": {"my_object": {"type": "object", "additionalProperties": "foo"}}, - } - ), - pytest.raises(AssertionError), - ), - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "additionalProperties": True, - "properties": { - "my_oneOf_object": {"type": "object", "oneOf": [{"additionalProperties": True}, {"additionalProperties": False}]} - }, - } - ), - pytest.raises(AssertionError), - ), - ], -) -def test_additional_properties_is_true(connector_spec, expectation): - t = _TestSpec() - with expectation: - t.test_additional_properties_is_true(connector_spec) diff --git a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md index ffaadea30ee0..b3b99d1310b3 100644 --- a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md @@ -98,8 +98,8 @@ Verify that a spec operation issued to the connector returns a valid spec. | Input | Type | Default | Note | | :--- | :--- | :--- |:-------------------------------------------------------------------------------------------------| | `spec_path` | string | `secrets/spec.json` | Path to a YAML or JSON file representing the spec expected to be output by this connector | -| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests. | -| `backward_compatibility_tests_config.run_backward_compatibility_tests` | boolean | True | Flag to run or skip backward compatibility tests. | +| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests (expects a version following semantic versioning). | +| `backward_compatibility_tests_config.disable_for_version` | string | None | Disable the backward compatibility test for a specific version (expects a version following semantic versioning). | | `timeout_seconds` | int | 10 | Test execution timeout in seconds | ## Test Connection From a4e52cc56403c6bf7d3eb68f373fdd16138428bf Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Fri, 5 Aug 2022 10:04:08 +0300 Subject: [PATCH 02/25] =?UTF-8?q?=F0=9F=90=9B=20Source=20Amazon=20Ads:=20I?= =?UTF-8?q?mprove=20report=20streams=20date-range=20generation=20(#15031)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergey Chvalyuk --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 6 +- .../connectors/source-amazon-ads/Dockerfile | 2 +- .../connectors/source-amazon-ads/README.md | 2 +- .../acceptance-test-config.yml | 44 ++--- .../acceptance-test-docker.sh | 2 +- .../integration_tests/abnormal_state.json | 7 + .../integration_tests/configured_catalog.json | 65 ++++++- .../configured_catalog_report.json | 14 ++ .../configured_catalog_sponsored_display.json | 44 ----- .../integration_tests/expected_records.txt | 145 +++++++++++--- .../expected_records_sponsored_display.txt | 6 - .../integration_tests/spec.json | 4 +- .../source-amazon-ads/requirements.txt | 1 + .../connectors/source-amazon-ads/setup.py | 2 +- .../source_amazon_ads/constants.py | 2 - .../source_amazon_ads/schemas/common.py | 2 + .../source_amazon_ads/spec.yaml | 3 +- .../streams/report_streams/products_report.py | 2 +- .../streams/report_streams/report_streams.py | 184 ++++++++---------- .../unit_tests/test_report_streams.py | 98 ++++++---- docs/integrations/sources/amazon-ads.md | 7 +- 22 files changed, 386 insertions(+), 258 deletions(-) create mode 100644 airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json delete mode 100644 airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_sponsored_display.json delete mode 100644 airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records_sponsored_display.txt diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 70173914da9c..9be4fb2490f0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -17,7 +17,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index fd2a3891239f..65e484cebc24 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -87,7 +87,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.10" +- dockerImage: "airbyte/source-amazon-ads:0.1.11" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: @@ -123,14 +123,12 @@ type: "string" region: title: "Region *" - description: "Region to pull data from (EU/NA/FE/SANDBOX). See docs for more details." enum: - "NA" - "EU" - "FE" - - "SANDBOX" type: "string" default: "NA" order: 4 diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 62557b694d3e..dac5a675caec 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/README.md b/airbyte-integrations/connectors/source-amazon-ads/README.md index 18da21bbe9ac..3002f939b404 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/README.md +++ b/airbyte-integrations/connectors/source-amazon-ads/README.md @@ -8,7 +8,7 @@ For information about how to use this connector within Airbyte, see [the documen ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +#### Minimum Python version required `= 3.9.0` #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index 24644c5c3e2a..1e0dbb030227 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -5,43 +5,35 @@ tests: spec: - spec_path: "integration_tests/spec.json" connection: - # THIS TEST IS COMMENTED OUT BECAUSE OF - # https://advertising.amazon.com/API/docs/en-us/info/release-notes#sandbox-deprecation-on-june-28-2022 - # - config_path: "secrets/config.json" - # status: "succeed" - - config_path: "secrets/config_test_account.json" + - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: - # THIS TEST IS COMMENTED OUT BECAUSE OF LOST ACCESS TO SANDBOX - # - config_path: "secrets/config.json" - - config_path: "secrets/config_test_account.json" + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config_test_account.json" + - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["sponsored_product_targetings"] + empty_streams: ["sponsored_brands_campaigns", "sponsored_brands_ad_groups", "sponsored_brands_keywords"] expect_records: path: "integration_tests/expected_records.txt" extra_fields: no exact_order: no extra_records: no - timeout_seconds: 3600 - # THIS TEST IS COMMENTED OUT BECAUSE OF - # https://advertising.amazon.com/API/docs/en-us/info/release-notes#sandbox-deprecation-on-june-28-2022 - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog_sponsored_display.json" - # empty_streams: ["sponsored_display_targetings"] - # expect_records: - # path: "integration_tests/expected_records_sponsored_display.txt" - # extra_fields: no - # exact_order: no - # extra_records: no + - config_path: "secrets/config_report.json" + configured_catalog_path: "integration_tests/configured_catalog_report.json" + timeout_seconds: 2400 + incremental: + - config_path: "secrets/config_report.json" + configured_catalog_path: "integration_tests/configured_catalog_report.json" + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + sponsored_products_report_stream: ["1861552880916640", "reportDate"] full_refresh: - - config_path: "secrets/config_test_account.json" + - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_report.json" + configured_catalog_path: "integration_tests/configured_catalog_report.json" + ignored_fields: + "sponsored_products_report_stream": ["updatedAt"] timeout_seconds: 3600 - # THIS TEST IS COMMENTED OUT BECAUSE OF - # https://advertising.amazon.com/API/docs/en-us/info/release-notes#sandbox-deprecation-on-june-28-2022 - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog_sponsored_display.json" diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh index e4d8b1cef896..c51577d10690 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..cef25db0669a --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json @@ -0,0 +1,7 @@ +{ + "sponsored_products_report_stream": { + "1861552880916640": { + "reportDate": "20990101" + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json index 1118c80fd871..31e97a02276f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json @@ -10,6 +10,46 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "sponsored_display_campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["campaignId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sponsored_display_ad_groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["adGroupId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sponsored_display_product_ads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["adId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sponsored_display_targetings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["targetId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "sponsored_product_campaigns", @@ -72,9 +112,30 @@ }, { "stream": { - "name": "sponsored_products_report_stream", + "name": "sponsored_brands_campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["campaignId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sponsored_brands_ad_groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["adGroupId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sponsored_brands_keywords", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["adGroupId"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json new file mode 100644 index 000000000000..26d697a48e1d --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "stream": { + "name": "sponsored_products_report_stream", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["profileId"], ["recordType"], ["reportDate"], ["updatedAt"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_sponsored_display.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_sponsored_display.json deleted file mode 100644 index f5526743fed0..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_sponsored_display.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "sponsored_display_campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["campaignId"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "sponsored_display_ad_groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["adGroupId"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "sponsored_display_product_ads", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["adId"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "sponsored_display_targetings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["targetId"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt index a6b1d053dc82..d74c774f8342 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt @@ -1,23 +1,122 @@ -{"stream":"profiles","data":{"profileId":3991703629696934,"countryCode":"CA","currencyCode":"CAD","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A2EUQ1WTGCTBG2","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1657279857756} -{"stream":"profiles","data":{"profileId":2935840597082037,"countryCode":"CA","currencyCode":"CAD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A2EUQ1WTGCTBG2","id":"ENTITY1T4PQ8E0Y1LVJ","type":"vendor","name":"test","validPaymentMethod":false}},"emitted_at":1657279857758} -{"stream":"profiles","data":{"profileId":3664951271230581,"countryCode":"MX","currencyCode":"MXN","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A1AM78C64UM0Y8","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1657279857758} -{"stream":"profiles","data":{"profileId":1861552880916640,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITYVFIQ1E6W9INI","type":"vendor","name":"Sponsored ads - KDP","subType":"KDP_AUTHOR","validPaymentMethod":true}},"emitted_at":1657279857759} -{"stream":"profiles","data":{"profileId":3312910465837761,"countryCode":"US","currencyCode":"USD","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1657279857759} -{"stream":"profiles","data":{"profileId":3039403378822505,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITY2ZP3PPFBG2043","type":"vendor","name":"3PTestBrand-A3LUQZ2NBMFGO4215634471126","validPaymentMethod":true}},"emitted_at":1657279857759} -{"stream":"profiles","data":{"profileId":2445745172318948,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITY3QRPN1GHC1Q0U","type":"vendor","name":"3PTestBrand-A3LUQZ2NBMFGO46750119612846","validPaymentMethod":true}},"emitted_at":1657791275850} -{"stream":"sponsored_product_campaigns","data":{"campaignId":39413387973397,"name":"Test campaging for profileId 1861552880916640","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":10,"ruleBasedBudget":{"isProcessing":false},"startDate":"20220705","endDate":"20220712","state":"paused","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1657796231723} -{"stream":"sponsored_product_ad_groups","data":{"adGroupId":226404883721634,"name":"My AdGroup for Campaign 39413387973397","campaignId":39413387973397,"defaultBid":10,"state":"enabled"},"emitted_at":1657279859462} -{"stream":"sponsored_product_keywords","data":{"keywordId":88368653576677,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"keyword1","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1657279860281} -{"stream":"sponsored_product_keywords","data":{"keywordId":256414981667762,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"broad","state":"enabled","bid":1.12},"emitted_at":1657279860282} -{"stream":"sponsored_product_keywords","data":{"keywordId":162522197737998,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"phrase","state":"enabled","bid":2.85},"emitted_at":1657279860283} -{"stream":"sponsored_product_keywords","data":{"keywordId":156474025571250,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1657279860283} -{"stream":"sponsored_product_keywords","data":{"keywordId":97960974522677,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"broad","state":"enabled","bid":0.83},"emitted_at":1657279860283} -{"stream":"sponsored_product_keywords","data":{"keywordId":21494218191267,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"phrase","state":"enabled","bid":4.06},"emitted_at":1657279860284} -{"stream":"sponsored_product_keywords","data":{"keywordId":122265145299463,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1657279860284} -{"stream":"sponsored_product_keywords","data":{"keywordId":105707339702386,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"broad","state":"enabled","bid":3.52},"emitted_at":1657279860284} -{"stream":"sponsored_product_keywords","data":{"keywordId":185938124401124,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"phrase","state":"enabled","bid":3.44},"emitted_at":1657279860284} -{"stream":"sponsored_product_keywords","data":{"keywordId":16455263285469,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"exact","state":"enabled","bid":3.69},"emitted_at":1657279860285} -{"stream":"sponsored_product_negative_keywords","data":{"keywordId":32531566025493,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"negkeyword1","matchType":"negativeExact","state":"enabled"},"emitted_at":1657279861100} -{"stream":"sponsored_product_ads","data":{"adId":134721479349712,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3NTQ5S","state":"enabled"},"emitted_at":1657279861919} -{"stream":"sponsored_product_ads","data":{"adId":265970953521535,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3QCS24","state":"enabled"},"emitted_at":1657279861920} -{"stream":"sponsored_product_ads","data":{"adId":253366527049144,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3P7D6Z","state":"enabled"},"emitted_at":1657279861920} +{"stream":"profiles","data":{"profileId":3991703629696934,"countryCode":"CA","currencyCode":"CAD","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A2EUQ1WTGCTBG2","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1659020216524} +{"stream":"profiles","data":{"profileId":2935840597082037,"countryCode":"CA","currencyCode":"CAD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A2EUQ1WTGCTBG2","id":"ENTITY1T4PQ8E0Y1LVJ","type":"vendor","name":"test","validPaymentMethod":false}},"emitted_at":1659020216526} +{"stream":"profiles","data":{"profileId":3664951271230581,"countryCode":"MX","currencyCode":"MXN","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A1AM78C64UM0Y8","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1659020216527} +{"stream":"profiles","data":{"profileId":1861552880916640,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITYVFIQ1E6W9INI","type":"vendor","name":"Sponsored ads - KDP","subType":"KDP_AUTHOR","validPaymentMethod":true}},"emitted_at":1659020216527} +{"stream":"profiles","data":{"profileId":3312910465837761,"countryCode":"US","currencyCode":"USD","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1659020216527} +{"stream":"profiles","data":{"profileId":2445745172318948,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITY3QRPN1GHC1Q0U","type":"vendor","name":"3PTestBrand-A3LUQZ2NBMFGO46750119612846","validPaymentMethod":true}},"emitted_at":1659020216527} +{"stream":"profiles","data":{"profileId":3039403378822505,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITY2ZP3PPFBG2043","type":"vendor","name":"3PTestBrand-A3LUQZ2NBMFGO4215634471126","validPaymentMethod":true}},"emitted_at":1659020216528} +{"stream":"sponsored_display_campaigns","data":{"campaignId":25934734632378,"name":"Campaign - 7/20/2022 15:45:46","tactic":"T00020","startDate":"20240510","state":"enabled","costType":"cpc","budget":1,"budgetType":"daily","deliveryProfile":"as_soon_as_possible"},"emitted_at":1659020217679} +{"stream":"sponsored_display_ad_groups","data":{"adGroupId":239470166910761,"campaignId":25934734632378,"defaultBid":0.02,"name":"Ad group - 7/20/2022 15:45:46","state":"enabled","bidOptimization":"clicks","tactic":"T00020"},"emitted_at":1659020218593} +{"stream":"sponsored_display_product_ads","data":{"adId":125773733335504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT390","state":"enabled"},"emitted_at":1659020219604} +{"stream":"sponsored_display_product_ads","data":{"adId":22923447445879,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJK","state":"enabled"},"emitted_at":1659020219605} +{"stream":"sponsored_display_product_ads","data":{"adId":174434781640143,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B006K1JR0W","state":"enabled"},"emitted_at":1659020219605} +{"stream":"sponsored_display_product_ads","data":{"adId":209576432984926,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58W","state":"enabled"},"emitted_at":1659020219605} +{"stream":"sponsored_display_product_ads","data":{"adId":78757678617297,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0BD0","state":"enabled"},"emitted_at":1659020219606} +{"stream":"sponsored_display_product_ads","data":{"adId":193756923178712,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNG","state":"enabled"},"emitted_at":1659020219606} +{"stream":"sponsored_display_product_ads","data":{"adId":31271769792588,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT38G","state":"enabled"},"emitted_at":1659020219606} +{"stream":"sponsored_display_product_ads","data":{"adId":150153237605370,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV596","state":"enabled"},"emitted_at":1659020219606} +{"stream":"sponsored_display_product_ads","data":{"adId":2074333536480,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0R66","state":"enabled"},"emitted_at":1659020219607} +{"stream":"sponsored_display_product_ads","data":{"adId":123533571549424,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2O","state":"enabled"},"emitted_at":1659020219607} +{"stream":"sponsored_display_product_ads","data":{"adId":217260138761504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091FZ92NV","state":"enabled"},"emitted_at":1659020219607} +{"stream":"sponsored_display_product_ads","data":{"adId":145457886517316,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD1U","state":"enabled"},"emitted_at":1659020219607} +{"stream":"sponsored_display_product_ads","data":{"adId":203822232798249,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E9VEK","state":"enabled"},"emitted_at":1659020219608} +{"stream":"sponsored_display_product_ads","data":{"adId":117735697461953,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNQ","state":"enabled"},"emitted_at":1659020219608} +{"stream":"sponsored_display_product_ads","data":{"adId":142089319699283,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G29WN9","state":"enabled"},"emitted_at":1659020219608} +{"stream":"sponsored_display_product_ads","data":{"adId":95431347262692,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1JCM","state":"enabled"},"emitted_at":1659020219608} +{"stream":"sponsored_display_product_ads","data":{"adId":155014902487440,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJU","state":"enabled"},"emitted_at":1659020219609} +{"stream":"sponsored_display_product_ads","data":{"adId":11743222321360,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AE","state":"enabled"},"emitted_at":1659020219609} +{"stream":"sponsored_display_product_ads","data":{"adId":103439653344998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00RW78E52","state":"enabled"},"emitted_at":1659020219609} +{"stream":"sponsored_display_product_ads","data":{"adId":265969657657801,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39K","state":"enabled"},"emitted_at":1659020219609} +{"stream":"sponsored_display_product_ads","data":{"adId":109412610635634,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39U","state":"enabled"},"emitted_at":1659020219610} +{"stream":"sponsored_display_product_ads","data":{"adId":136393331771998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV59Q","state":"enabled"},"emitted_at":1659020219610} +{"stream":"sponsored_display_product_ads","data":{"adId":186420999434919,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2E","state":"enabled"},"emitted_at":1659020219610} +{"stream":"sponsored_display_product_ads","data":{"adId":278853238562368,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G35ZDQ","state":"enabled"},"emitted_at":1659020219610} +{"stream":"sponsored_display_product_ads","data":{"adId":166899201791771,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1SLE","state":"enabled"},"emitted_at":1659020219611} +{"stream":"sponsored_display_product_ads","data":{"adId":109280751164007,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G3QCHL","state":"enabled"},"emitted_at":1659020219611} +{"stream":"sponsored_display_product_ads","data":{"adId":151372475824008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39A","state":"enabled"},"emitted_at":1659020219611} +{"stream":"sponsored_display_product_ads","data":{"adId":111491538035732,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00CKZKG20","state":"enabled"},"emitted_at":1659020219611} +{"stream":"sponsored_display_product_ads","data":{"adId":61045475129398,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E3HUO","state":"enabled"},"emitted_at":1659020219611} +{"stream":"sponsored_display_product_ads","data":{"adId":125617015283672,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBO0","state":"enabled"},"emitted_at":1659020219612} +{"stream":"sponsored_display_product_ads","data":{"adId":183608040922804,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBMW","state":"enabled"},"emitted_at":1659020219612} +{"stream":"sponsored_display_product_ads","data":{"adId":252975632234287,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58M","state":"enabled"},"emitted_at":1659020219612} +{"stream":"sponsored_display_product_ads","data":{"adId":223374763750850,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2Y","state":"enabled"},"emitted_at":1659020219612} +{"stream":"sponsored_display_product_ads","data":{"adId":155052344322362,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37M","state":"enabled"},"emitted_at":1659020219613} +{"stream":"sponsored_display_product_ads","data":{"adId":210510170479158,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AY","state":"enabled"},"emitted_at":1659020219613} +{"stream":"sponsored_display_product_ads","data":{"adId":179517989169690,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37W","state":"enabled"},"emitted_at":1659020219613} +{"stream":"sponsored_display_product_ads","data":{"adId":163992879107492,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV5AA","state":"enabled"},"emitted_at":1659020219613} +{"stream":"sponsored_display_product_ads","data":{"adId":103527738992867,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT386","state":"enabled"},"emitted_at":1659020219614} +{"stream":"sponsored_display_product_ads","data":{"adId":195948665185008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBOA","state":"enabled"},"emitted_at":1659020219614} +{"stream":"sponsored_display_product_ads","data":{"adId":130802512011075,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G1HT4P","state":"enabled"},"emitted_at":1659020219614} +{"stream":"sponsored_display_targetings","data":{"adGroupId":239470166910761,"bid":0.4,"expression":[{"type":"similarProduct"}],"expressionType":"auto","resolvedExpression":[{"type":"similarProduct"}],"state":"enabled","targetId":124150067548052},"emitted_at":1659020220625} +{"stream":"sponsored_product_campaigns","data":{"campaignId":39413387973397,"name":"Test campaging for profileId 1861552880916640","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":10,"ruleBasedBudget":{"isProcessing":false},"startDate":"20220705","endDate":"20220712","state":"paused","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1659020221212} +{"stream":"sponsored_product_campaigns","data":{"campaignId":135264288913079,"name":"Campaign - 7/5/2022 18:14:02","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":false,"dailyBudget":10,"startDate":"20220705","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[]}},"emitted_at":1659020221384} +{"stream":"sponsored_product_campaigns","data":{"campaignId":191249325250025,"name":"Campaign - 7/8/2022 13:57:48","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":true,"dailyBudget":50,"startDate":"20220708","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":100},{"predicate":"placementProductPage","percentage":100}]}},"emitted_at":1659020221384} +{"stream":"sponsored_product_campaigns","data":{"campaignId":146003174711486,"name":"Test campaging for profileId 3039403378822505","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":1,"startDate":"20220705","endDate":"20231111","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1659020221553} +{"stream":"sponsored_product_ad_groups","data":{"adGroupId":226404883721634,"name":"My AdGroup for Campaign 39413387973397","campaignId":39413387973397,"defaultBid":10,"state":"enabled"},"emitted_at":1659020222108} +{"stream":"sponsored_product_ad_groups","data":{"adGroupId":183961953969922,"name":"Ad group - 7/5/2022 18:14:02","campaignId":135264288913079,"defaultBid":0.75,"state":"enabled"},"emitted_at":1659020222276} +{"stream":"sponsored_product_ad_groups","data":{"adGroupId":108551155050351,"name":"Ad group - 7/8/2022 13:57:48","campaignId":191249325250025,"defaultBid":1,"state":"enabled"},"emitted_at":1659020222276} +{"stream":"sponsored_product_ad_groups","data":{"adGroupId":103188883625219,"name":"My AdGroup for Campaign 146003174711486","campaignId":146003174711486,"defaultBid":10,"state":"enabled"},"emitted_at":1659020222593} +{"stream":"sponsored_product_keywords","data":{"keywordId":88368653576677,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"keyword1","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1659020223173} +{"stream":"sponsored_product_keywords","data":{"keywordId":256414981667762,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"broad","state":"enabled","bid":1.12},"emitted_at":1659020223174} +{"stream":"sponsored_product_keywords","data":{"keywordId":162522197737998,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"phrase","state":"enabled","bid":2.85},"emitted_at":1659020223175} +{"stream":"sponsored_product_keywords","data":{"keywordId":156474025571250,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1659020223175} +{"stream":"sponsored_product_keywords","data":{"keywordId":97960974522677,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"broad","state":"enabled","bid":0.83},"emitted_at":1659020223175} +{"stream":"sponsored_product_keywords","data":{"keywordId":21494218191267,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"phrase","state":"enabled","bid":4.06},"emitted_at":1659020223175} +{"stream":"sponsored_product_keywords","data":{"keywordId":122265145299463,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1659020223176} +{"stream":"sponsored_product_keywords","data":{"keywordId":105707339702386,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"broad","state":"enabled","bid":3.52},"emitted_at":1659020223176} +{"stream":"sponsored_product_keywords","data":{"keywordId":185938124401124,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"phrase","state":"enabled","bid":3.44},"emitted_at":1659020223176} +{"stream":"sponsored_product_keywords","data":{"keywordId":16455263285469,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"exact","state":"enabled","bid":3.69},"emitted_at":1659020223177} +{"stream":"sponsored_product_negative_keywords","data":{"keywordId":32531566025493,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"negkeyword1","matchType":"negativeExact","state":"enabled"},"emitted_at":1659020224091} +{"stream":"sponsored_product_ads","data":{"adId":134721479349712,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3NTQ5S","state":"enabled"},"emitted_at":1659020225056} +{"stream":"sponsored_product_ads","data":{"adId":265970953521535,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3QCS24","state":"enabled"},"emitted_at":1659020225057} +{"stream":"sponsored_product_ads","data":{"adId":253366527049144,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3P7D6Z","state":"enabled"},"emitted_at":1659020225057} +{"stream":"sponsored_product_ads","data":{"adId":44137758141732,"adGroupId":183961953969922,"campaignId":135264288913079,"asin":"B000VHYM2E","sku":"0R-4KDA-Z2U8","state":"enabled"},"emitted_at":1659020225248} +{"stream":"sponsored_product_ads","data":{"adId":126456292487945,"adGroupId":108551155050351,"campaignId":191249325250025,"asin":"B074K5MDLW","sku":"2J-D6V7-C8XI","state":"enabled"},"emitted_at":1659020225248} +{"stream":"sponsored_product_ads","data":{"adId":125773733335504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT390","state":"enabled"},"emitted_at":1659020225461} +{"stream":"sponsored_product_ads","data":{"adId":22923447445879,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJK","state":"enabled"},"emitted_at":1659020225461} +{"stream":"sponsored_product_ads","data":{"adId":174434781640143,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B006K1JR0W","state":"enabled"},"emitted_at":1659020225462} +{"stream":"sponsored_product_ads","data":{"adId":209576432984926,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58W","state":"enabled"},"emitted_at":1659020225462} +{"stream":"sponsored_product_ads","data":{"adId":78757678617297,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0BD0","state":"enabled"},"emitted_at":1659020225462} +{"stream":"sponsored_product_ads","data":{"adId":193756923178712,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNG","state":"enabled"},"emitted_at":1659020225462} +{"stream":"sponsored_product_ads","data":{"adId":31271769792588,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT38G","state":"enabled"},"emitted_at":1659020225463} +{"stream":"sponsored_product_ads","data":{"adId":150153237605370,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV596","state":"enabled"},"emitted_at":1659020225463} +{"stream":"sponsored_product_ads","data":{"adId":2074333536480,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0R66","state":"enabled"},"emitted_at":1659020225463} +{"stream":"sponsored_product_ads","data":{"adId":123533571549424,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2O","state":"enabled"},"emitted_at":1659020225463} +{"stream":"sponsored_product_ads","data":{"adId":217260138761504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091FZ92NV","state":"enabled"},"emitted_at":1659020225464} +{"stream":"sponsored_product_ads","data":{"adId":145457886517316,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD1U","state":"enabled"},"emitted_at":1659020225464} +{"stream":"sponsored_product_ads","data":{"adId":203822232798249,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E9VEK","state":"enabled"},"emitted_at":1659020225464} +{"stream":"sponsored_product_ads","data":{"adId":117735697461953,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNQ","state":"enabled"},"emitted_at":1659020225464} +{"stream":"sponsored_product_ads","data":{"adId":142089319699283,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G29WN9","state":"enabled"},"emitted_at":1659020225465} +{"stream":"sponsored_product_ads","data":{"adId":95431347262692,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1JCM","state":"enabled"},"emitted_at":1659020225465} +{"stream":"sponsored_product_ads","data":{"adId":155014902487440,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJU","state":"enabled"},"emitted_at":1659020225465} +{"stream":"sponsored_product_ads","data":{"adId":11743222321360,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AE","state":"enabled"},"emitted_at":1659020225465} +{"stream":"sponsored_product_ads","data":{"adId":103439653344998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00RW78E52","state":"enabled"},"emitted_at":1659020225466} +{"stream":"sponsored_product_ads","data":{"adId":265969657657801,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39K","state":"enabled"},"emitted_at":1659020225466} +{"stream":"sponsored_product_ads","data":{"adId":109412610635634,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39U","state":"enabled"},"emitted_at":1659020225466} +{"stream":"sponsored_product_ads","data":{"adId":136393331771998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV59Q","state":"enabled"},"emitted_at":1659020225466} +{"stream":"sponsored_product_ads","data":{"adId":186420999434919,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2E","state":"enabled"},"emitted_at":1659020225467} +{"stream":"sponsored_product_ads","data":{"adId":278853238562368,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G35ZDQ","state":"enabled"},"emitted_at":1659020225467} +{"stream":"sponsored_product_ads","data":{"adId":166899201791771,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1SLE","state":"enabled"},"emitted_at":1659020225467} +{"stream":"sponsored_product_ads","data":{"adId":109280751164007,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G3QCHL","state":"enabled"},"emitted_at":1659020225467} +{"stream":"sponsored_product_ads","data":{"adId":151372475824008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39A","state":"enabled"},"emitted_at":1659020225467} +{"stream":"sponsored_product_ads","data":{"adId":111491538035732,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00CKZKG20","state":"enabled"},"emitted_at":1659020225468} +{"stream":"sponsored_product_ads","data":{"adId":61045475129398,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E3HUO","state":"enabled"},"emitted_at":1659020225468} +{"stream":"sponsored_product_ads","data":{"adId":125617015283672,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBO0","state":"enabled"},"emitted_at":1659020225468} +{"stream":"sponsored_product_ads","data":{"adId":183608040922804,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBMW","state":"enabled"},"emitted_at":1659020225468} +{"stream":"sponsored_product_ads","data":{"adId":252975632234287,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58M","state":"enabled"},"emitted_at":1659020225469} +{"stream":"sponsored_product_ads","data":{"adId":223374763750850,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2Y","state":"enabled"},"emitted_at":1659020225469} +{"stream":"sponsored_product_ads","data":{"adId":155052344322362,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37M","state":"enabled"},"emitted_at":1659020225469} +{"stream":"sponsored_product_ads","data":{"adId":210510170479158,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AY","state":"enabled"},"emitted_at":1659020225470} +{"stream":"sponsored_product_ads","data":{"adId":179517989169690,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37W","state":"enabled"},"emitted_at":1659020225470} +{"stream":"sponsored_product_ads","data":{"adId":163992879107492,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV5AA","state":"enabled"},"emitted_at":1659020225470} +{"stream":"sponsored_product_ads","data":{"adId":103527738992867,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT386","state":"enabled"},"emitted_at":1659020225470} +{"stream":"sponsored_product_ads","data":{"adId":195948665185008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBOA","state":"enabled"},"emitted_at":1659020225470} +{"stream":"sponsored_product_ads","data":{"adId":130802512011075,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G1HT4P","state":"enabled"},"emitted_at":1659020225471} +{"stream":"sponsored_product_targetings","data":{"targetId":50319181484813,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"queryHighRelMatches"}],"resolvedExpression":[{"type":"queryHighRelMatches"}]},"emitted_at":1659020226434} +{"stream":"sponsored_product_targetings","data":{"targetId":27674318672023,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"queryBroadRelMatches"}],"resolvedExpression":[{"type":"queryBroadRelMatches"}]},"emitted_at":1659020226435} +{"stream":"sponsored_product_targetings","data":{"targetId":231060819625654,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"asinAccessoryRelated"}],"resolvedExpression":[{"type":"asinAccessoryRelated"}]},"emitted_at":1659020226435} +{"stream":"sponsored_product_targetings","data":{"targetId":223980840024498,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"asinSubstituteRelated"}],"resolvedExpression":[{"type":"asinSubstituteRelated"}]},"emitted_at":1659020226436} +{"stream":"sponsored_product_targetings","data":{"targetId":62579800516352,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"queryHighRelMatches"}],"resolvedExpression":[{"type":"queryHighRelMatches"}]},"emitted_at":1659020226436} +{"stream":"sponsored_product_targetings","data":{"targetId":232221427954900,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"queryBroadRelMatches"}],"resolvedExpression":[{"type":"queryBroadRelMatches"}]},"emitted_at":1659020226436} +{"stream":"sponsored_product_targetings","data":{"targetId":12739477778779,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"asinAccessoryRelated"}],"resolvedExpression":[{"type":"asinAccessoryRelated"}]},"emitted_at":1659020226436} +{"stream":"sponsored_product_targetings","data":{"targetId":1189452552122,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"asinSubstituteRelated"}],"resolvedExpression":[{"type":"asinSubstituteRelated"}]},"emitted_at":1659020226437} diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records_sponsored_display.txt b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records_sponsored_display.txt deleted file mode 100644 index e21d5abc6368..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records_sponsored_display.txt +++ /dev/null @@ -1,6 +0,0 @@ -{"stream":"sponsored_display_campaigns","data":{"campaignId":37387403419888,"name":"sswdd","tactic":"T00020","startDate":"20220101","state":"enabled","costType":"cpc","budget":3,"budgetType":"daily","deliveryProfile":"as_soon_as_possible"},"emitted_at":1649053342947} -{"stream":"sponsored_display_campaigns","data":{"campaignId":59249214322256,"name":"My test camp","tactic":"T00020","startDate":"20220101","state":"enabled","costType":"cpc","budget":3,"budgetType":"daily","deliveryProfile":"as_soon_as_possible"},"emitted_at":1649053342949} -{"stream":"sponsored_display_campaigns","data":{"campaignId":16117299922278,"name":"ssw","tactic":"T00020","startDate":"20220101","state":"enabled","costType":"cpc","budget":3,"budgetType":"daily","deliveryProfile":"as_soon_as_possible"},"emitted_at":1649053342950} -{"stream":"sponsored_display_campaigns","data":{"campaignId":202914386115504,"name":"ssdf","tactic":"T00020","startDate":"20220101","state":"enabled","costType":"cpc","budget":3,"budgetType":"daily","deliveryProfile":"as_soon_as_possible"},"emitted_at":1649053342950} -{"stream":"sponsored_display_ad_groups","data":{"adGroupId":154135351589329,"campaignId":59249214322256,"defaultBid":0.02,"name":"string","state":"enabled","bidOptimization":"clicks","tactic":"T00020"},"emitted_at":1649053344279} -{"stream":"sponsored_display_product_ads","data":{"adId":95299121316520,"adGroupId":154135351589329,"campaignId":59249214322256,"sku":"string","state":"enabled"},"emitted_at":1649053345472} diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 087b1de5be57..cf6970808b69 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -32,8 +32,8 @@ }, "region": { "title": "Region *", - "description": "Region to pull data from (EU/NA/FE/SANDBOX). See docs for more details.", - "enum": ["NA", "EU", "FE", "SANDBOX"], + "description": "Region to pull data from (EU/NA/FE). See docs for more details.", + "enum": ["NA", "EU", "FE"], "type": "string", "default": "NA", "order": 4 diff --git a/airbyte-integrations/connectors/source-amazon-ads/requirements.txt b/airbyte-integrations/connectors/source-amazon-ads/requirements.txt index 0411042aa091..7be17a56d745 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/requirements.txt +++ b/airbyte-integrations/connectors/source-amazon-ads/requirements.txt @@ -1,2 +1,3 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py index 3c7d5d5e9732..5d9c4a766fd8 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/setup.py +++ b/airbyte-integrations/connectors/source-amazon-ads/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.46", "requests_oauthlib~=1.3.0", "pytz~=2021.1", "pendulum~=2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.68", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py index 0d422403b0bb..7b7251b87548 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/constants.py @@ -9,12 +9,10 @@ class AmazonAdsRegion(str, Enum): NA = "NA" EU = "EU" FE = "FE" - SANDBOX = "SANDBOX" URL_MAPPING = { "NA": "https://advertising-api.amazon.com/", "EU": "https://advertising-api-eu.amazon.com/", "FE": "https://advertising-api-fe.amazon.com/", - "SANDBOX": "https://advertising-api-test.amazon.com/", } diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py index be68719ae67b..fb97b9eae68f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from datetime import datetime from decimal import Decimal from typing import Any, Dict, Iterable, Type @@ -42,6 +43,7 @@ class MetricsReport(CatalogModel): profileId: int recordType: str reportDate: str + updatedAt: datetime # This property will be overwritten with autogenerated model based on metrics list metric: None diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index 937444318428..c0d2af11b0a1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -37,13 +37,12 @@ connectionSpecification: region: title: Region * description: - Region to pull data from (EU/NA/FE/SANDBOX). See docs + Region to pull data from (EU/NA/FE). See docs for more details. enum: - NA - EU - FE - - SANDBOX type: string default: NA order: 4 diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index b81141a95d7c..9f36e185ee78 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -248,7 +248,7 @@ class SponsoredProductsReportStream(ReportStream): https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Reports """ - primary_key = ["profileId", "recordType", "reportDate"] + primary_key = ["profileId", "recordType", "reportDate", "updatedAt"] def report_init_endpoint(self, record_type: str) -> str: return f"/v2/sp/{record_type}/report" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 4ebaa733978f..93a2a3f6d2d8 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -5,7 +5,6 @@ import json from abc import ABC, abstractmethod from dataclasses import dataclass -from datetime import timedelta from enum import Enum from gzip import decompress from http import HTTPStatus @@ -14,12 +13,11 @@ import backoff import pendulum -import pytz import requests from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from pendulum import DateTime +from pendulum import Date from pydantic import BaseModel from source_amazon_ads.schemas import CatalogModel, MetricsReport, Profile from source_amazon_ads.streams.common import BasicAmazonAdsStream @@ -92,22 +90,24 @@ class ReportStream(BasicAmazonAdsStream, ABC): """ primary_key = None - CHECK_INTERVAL_SECONDS = 30 # Amazon ads updates the data for the next 3 days LOOK_BACK_WINDOW = 3 # (Service limits section) # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "YYYYMMDD" + CONFIG_DATE_FORMAT = "YYYY-MM-DD" cursor_field = "reportDate" def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator): self._authenticator = authenticator self._session = requests.Session() self._model = self._generate_model() - self.report_wait_timeout = timedelta(minutes=config.get("report_wait_timeout", 30)).total_seconds + self.report_wait_timeout = config.get("report_wait_timeout", 30) self.report_generation_maximum_retries = config.get("report_generation_max_retries", 5) - # Set start date from config file, should be in UTC timezone. - self._start_date = pendulum.parse(config.get("start_date")).set(tz="UTC") if config.get("start_date") else None + # Set start date from config file + self._start_date = config.get("start_date") + if self._start_date: + self._start_date = pendulum.from_format(self._start_date, self.CONFIG_DATE_FORMAT).date() super().__init__(config, profiles) @property @@ -137,8 +137,9 @@ def read_records( # parameter pointing to the future. In this case we dont need to # take any action and just return. return + profile = stream_slice["profile"] report_date = stream_slice[self.cursor_field] - report_infos = self._init_and_try_read_records(report_date) + report_infos = self._init_and_try_read_records(profile, report_date) for report_info in report_infos: for metric_object in report_info.metric_objects: @@ -146,12 +147,13 @@ def read_records( profileId=report_info.profile_id, recordType=report_info.record_type, reportDate=report_date, + updatedAt=pendulum.now(tz=profile.timezone).replace(microsecond=0).to_iso8601_string(), metric=metric_object, ).dict() def backoff_max_time(func): def wrapped(self, *args, **kwargs): - return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout)(func)( + return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout * 60, interval=10)(func)( self, *args, **kwargs ) @@ -166,8 +168,8 @@ def wrapped(self, *args, **kwargs): return wrapped @backoff_max_tries - def _init_and_try_read_records(self, report_date): - report_infos = self._init_reports(report_date) + def _init_and_try_read_records(self, profile: Profile, report_date): + report_infos = self._init_reports(profile, report_date) logger.info(f"Waiting for {len(report_infos)} report(s) to be generated") self._try_read_records(report_infos) return report_infos @@ -251,7 +253,7 @@ def _check_status(self, report_info: ReportInfo) -> Tuple[Status, str]: requests.exceptions.ConnectionError, TooManyRequests, ), - max_tries=5, + max_tries=10, ) def _send_http_request(self, url: str, profile_id: int, json: dict = None): headers = self._get_auth_headers(profile_id) @@ -263,51 +265,51 @@ def _send_http_request(self, url: str, profile_id: int, json: dict = None): raise TooManyRequests() return response - @staticmethod - def get_report_date_ranges(start_report_date: Optional[DateTime]) -> Iterable[str]: - """ - Generates dates in YYYYMMDD format for each day started from - start_report_date until current date (current date included) - :param start_report_date Starting date to generate report date list. In - case it is None it would return today's date. - :return List of days from start_report_date up until today in format - specified by REPORT_DATE_FORMAT variable. - """ - now = pendulum.now(tz="UTC") - if not start_report_date: - start_report_date = now + def get_date_range(self, start_date: Date, end_date: Date) -> Iterable[str]: + for days in range((end_date - start_date).days + 1): + yield start_date.add(days=days).format(ReportStream.REPORT_DATE_FORMAT) - # You cannot pull data for amazon ads more than 60 days - if (now - start_report_date).days > (60 - ReportStream.LOOK_BACK_WINDOW): - start_report_date = now + timedelta(days=-(60 - ReportStream.LOOK_BACK_WINDOW)) - - for days in range(0, (now - start_report_date).days + 1): - next_date = start_report_date + timedelta(days=days) - next_date = next_date.format(ReportStream.REPORT_DATE_FORMAT) - yield next_date + def get_start_date(self, profile: Profile, stream_state: Mapping[str, Any]) -> Date: + today = pendulum.today(tz=profile.timezone).date() + start_date = stream_state.get(str(profile.profileId), {}).get(self.cursor_field) + if start_date: + start_date = pendulum.from_format(start_date, self.REPORT_DATE_FORMAT).date() + return max(start_date, today.subtract(days=60)) + if self._start_date: + return max(self._start_date, today.subtract(days=60)) + return today def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - if sync_mode == SyncMode.full_refresh: - # For full refresh stream use date from config start_date field. - start_date = self._start_date - else: - # incremental stream - stream_state = stream_state or {} - start_date = stream_state.get(self.cursor_field) - if start_date: - start_date = pendulum.from_format(start_date, ReportStream.REPORT_DATE_FORMAT, tz="UTC") - start_date += timedelta(days=-ReportStream.LOOK_BACK_WINDOW) + stream_state = stream_state or {} - else: - start_date = self._start_date - - return [{self.cursor_field: date} for date in ReportStream.get_report_date_ranges(start_date)] or [None] + slices = [] + for profile in self._profiles: + today = pendulum.today(tz=profile.timezone).date() + start_date = self.get_start_date(profile, stream_state) + for report_date in self.get_date_range(start_date, today): + slices.append({"profile": profile, self.cursor_field: report_date}) + if not slices: + return [None] + return slices def get_updated_state(self, current_stream_state: Dict[str, Any], latest_data: Mapping[str, Any]) -> Mapping[str, Any]: - return {"reportDate": latest_data["reportDate"]} + profileId = str(latest_data["profileId"]) + profile = {str(p.profileId): p for p in self._profiles}[profileId] + record_date = latest_data[self.cursor_field] + record_date = pendulum.from_format(record_date, self.REPORT_DATE_FORMAT).date() + look_back_date = pendulum.today(tz=profile.timezone).date().subtract(days=self.LOOK_BACK_WINDOW) + start_date = self.get_start_date(profile, current_stream_state) + updated_state = max(min(record_date, look_back_date), start_date).format(self.REPORT_DATE_FORMAT) + + stream_state_value = current_stream_state.get(profileId, {}).get(self.cursor_field) + if stream_state_value: + updated_state = max(updated_state, stream_state_value) + current_stream_state.setdefault(profileId, {})[self.cursor_field] = updated_state + + return current_stream_state @abstractmethod def _get_init_report_body(self, report_date: str, record_type: str, profile) -> Dict[str, Any]: @@ -320,67 +322,48 @@ def _get_init_report_body(self, report_date: str, record_type: str, profile) -> ReportInitFailure, max_tries=5, ) - def _init_reports(self, report_date: str) -> List[ReportInfo]: + def _init_reports(self, profile: Profile, report_date: str) -> List[ReportInfo]: """ Send report generation requests for all profiles and for all record types for specific day. :report_date - date for generating metric report. :return List of ReportInfo objects each of them has reportId field to check report status. """ report_infos = [] - for profile in self._profiles: - for record_type, metrics in self.metrics_map.items(): - metric_date = self._calc_report_generation_date(report_date, profile) - - report_init_body = self._get_init_report_body(metric_date, record_type, profile) - if not report_init_body: - continue - # Some of the record types has subtypes. For example asins type - # for product report have keyword and targets subtypes and it - # repseneted as asins_keywords and asins_targets types. Those - # subtypes have mutualy excluded parameters so we requesting - # different metric list for each record. - record_type = record_type.split("_")[0] - logger.info(f"Initiating report generation for {profile.profileId} profile with {record_type} type for {metric_date} date") - response = self._send_http_request( - urljoin(self._url, self.report_init_endpoint(record_type)), - profile.profileId, - report_init_body, + for record_type, metrics in self.metrics_map.items(): + report_init_body = self._get_init_report_body(report_date, record_type, profile) + if not report_init_body: + continue + # Some of the record types has subtypes. For example asins type + # for product report have keyword and targets subtypes and it + # repseneted as asins_keywords and asins_targets types. Those + # subtypes have mutualy excluded parameters so we requesting + # different metric list for each record. + record_type = record_type.split("_")[0] + logger.info(f"Initiating report generation for {profile.profileId} profile with {record_type} type for {report_date} date") + response = self._send_http_request( + urljoin(self._url, self.report_init_endpoint(record_type)), + profile.profileId, + report_init_body, + ) + if response.status_code != HTTPStatus.ACCEPTED: + raise ReportInitFailure( + f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}" ) - if response.status_code != HTTPStatus.ACCEPTED: - raise ReportInitFailure( - f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}" - ) - - response = ReportInitResponse.parse_raw(response.text) - report_infos.append( - ReportInfo( - report_id=response.reportId, - record_type=record_type, - profile_id=profile.profileId, - status=Status.IN_PROGRESS, - metric_objects=[], - ) + + response = ReportInitResponse.parse_raw(response.text) + report_infos.append( + ReportInfo( + report_id=response.reportId, + record_type=record_type, + profile_id=profile.profileId, + status=Status.IN_PROGRESS, + metric_objects=[], ) - logger.info("Initiated successfully") + ) + logger.info("Initiated successfully") return report_infos - @staticmethod - def _calc_report_generation_date(report_date: str, profile) -> str: - """ - According to reference time zone is specified by the profile used to - request the report. If specified date is today, then the performance - report may contain partial information. Based on this we generating - reports from day before specified report date and we should take into - account timezone for each profile. - :param report_date requested date that stored in stream state. - :return date parameter for Amazon Ads generate report. - """ - report_date = pendulum.from_format(report_date, ReportStream.REPORT_DATE_FORMAT) - profile_tz = pytz.timezone(profile.timezone) - profile_time = report_date.astimezone(profile_tz) - return profile_time.format(ReportStream.REPORT_DATE_FORMAT) - @backoff.on_exception( backoff.expo, requests.HTTPError, @@ -394,3 +377,8 @@ def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: response.raise_for_status() raw_string = decompress(response.content).decode("utf") return json.loads(raw_string) + + def get_error_display_message(self, exception: BaseException) -> Optional[str]: + if isinstance(exception, ReportGenerationInProgress): + return f'Report(s) generation time took more than {self.report_wait_timeout} minutes, please increase the "report_wait_timeout" parameter in configuration.' + return super().get_error_display_message(exception) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 3edac2ffabc5..3bd230139e36 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -10,6 +10,7 @@ import responses from airbyte_cdk.models import SyncMode from freezegun import freeze_time +from pendulum import Date from pytest import raises from requests.exceptions import ConnectionError from source_amazon_ads.schemas.profile import AccountInfo, Profile @@ -114,14 +115,13 @@ def test_display_report_stream(config): profiles = make_profiles() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725"} + stream_slice = {"profile": profiles[0], "reportDate": "20210725"} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) - updated_state = stream.get_updated_state(None, stream_slice) - assert updated_state == stream_slice profiles = make_profiles(profile_type="vendor") stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) + stream_slice["profile"] = profiles[0] metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] # Skip asins record for vendor profiles assert len(metrics) == METRICS_COUNT * (len(stream.metrics_map) - 1) @@ -138,7 +138,7 @@ def test_products_report_stream(config): profiles = make_profiles(profile_type="vendor") stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725", "retry_count": 3} + stream_slice = {"profile": profiles[0], "reportDate": "20210725", "retry_count": 3} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) @@ -154,7 +154,7 @@ def test_brands_report_stream(config): profiles = make_profiles() stream = SponsoredBrandsReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725"} + stream_slice = {"profile": profiles[0], "reportDate": "20210725"} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) @@ -170,7 +170,7 @@ def test_brands_video_report_stream(config): profiles = make_profiles() stream = SponsoredBrandsVideoReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725"} + stream_slice = {"profile": profiles[0], "reportDate": "20210725"} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) @@ -179,7 +179,7 @@ def test_brands_video_report_stream(config): def test_display_report_stream_init_failure(mocker, config): profiles = make_profiles() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725"} + stream_slice = {"profile": profiles[0], "reportDate": "20210725"} responses.add( responses.POST, re.compile(r"https://advertising-api.amazon.com/sd/[a-zA-Z]+/report"), json={"error": "some error"}, status=400 ) @@ -197,12 +197,12 @@ def test_display_report_stream_init_http_exception(mocker, config): mocker.patch("time.sleep", lambda x: None) profiles = make_profiles() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725"} + stream_slice = {"profile": profiles[0], "reportDate": "20210725"} responses.add(responses.POST, re.compile(r"https://advertising-api.amazon.com/sd/[a-zA-Z]+/report"), body=ConnectionError()) with raises(ConnectionError): _ = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] - assert len(responses.calls) == 5 + assert len(responses.calls) == 10 @responses.activate @@ -210,12 +210,12 @@ def test_display_report_stream_init_too_many_requests(mocker, config): mocker.patch("time.sleep", lambda x: None) profiles = make_profiles() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725"} + stream_slice = {"profile": profiles[0], "reportDate": "20210725"} responses.add(responses.POST, re.compile(r"https://advertising-api.amazon.com/sd/[a-zA-Z]+/report"), json={}, status=429) with raises(TooManyRequests): _ = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] - assert len(responses.calls) == 5 + assert len(responses.calls) == 10 @pytest.mark.parametrize( @@ -286,7 +286,7 @@ def __call__(self, request): responses.add_callback(responses.GET, re.compile(r"https://advertising-api.amazon.com/v2/reports/[^/]+$"), callback=callback) profiles = make_profiles() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"reportDate": "20210725"} + stream_slice = {"profile": profiles[0], "reportDate": "20210725"} if isinstance(expected, int): list(stream.read_records(SyncMode.incremental, stream_slice=stream_slice)) @@ -299,46 +299,64 @@ def __call__(self, request): @freeze_time("2021-07-30 04:26:08") @responses.activate def test_display_report_stream_slices_full_refresh(config): - stream = SponsoredDisplayReportStream(config, None, authenticator=mock.MagicMock()) + profiles = make_profiles() + stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) slices = stream.stream_slices(SyncMode.full_refresh, cursor_field=stream.cursor_field) - assert slices == [{"reportDate": "20210730"}] + assert slices == [{"profile": profiles[0], "reportDate": "20210729"}] @freeze_time("2021-07-30 04:26:08") @responses.activate def test_display_report_stream_slices_incremental(config): - stream = SponsoredDisplayReportStream(config, None, authenticator=mock.MagicMock()) - stream_state = {"reportDate": "20210726"} - slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state) - assert slices == [ - {"reportDate": "20210723"}, - {"reportDate": "20210724"}, - {"reportDate": "20210725"}, - {"reportDate": "20210726"}, - {"reportDate": "20210727"}, - {"reportDate": "20210728"}, - {"reportDate": "20210729"}, - {"reportDate": "20210730"}, - ] - stream_state = {"reportDate": "20210730"} + profiles = make_profiles() + stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) + stream_state = {str(profiles[0].profileId): {"reportDate": "20210725"}} slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state) assert slices == [ - {"reportDate": "20210727"}, - {"reportDate": "20210728"}, - {"reportDate": "20210729"}, - {"reportDate": "20210730"}, + {"profile": profiles[0], "reportDate": "20210725"}, + {"profile": profiles[0], "reportDate": "20210726"}, + {"profile": profiles[0], "reportDate": "20210727"}, + {"profile": profiles[0], "reportDate": "20210728"}, + {"profile": profiles[0], "reportDate": "20210729"}, ] - stream_state = {"reportDate": "20210731"} + stream_state = {str(profiles[0].profileId): {"reportDate": "20210730"}} slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state=stream_state) - assert slices == [ - {"reportDate": "20210728"}, - {"reportDate": "20210729"}, - {"reportDate": "20210730"}, - ] + assert slices == [None] slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state={}) - assert slices == [{"reportDate": "20210730"}] + assert slices == [{"profile": profiles[0], "reportDate": "20210729"}] slices = stream.stream_slices(SyncMode.incremental, cursor_field=None, stream_state={}) - assert slices == [{"reportDate": "20210730"}] + assert slices == [{"profile": profiles[0], "reportDate": "20210729"}] + + +@freeze_time("2021-08-01 04:00:00") +def test_get_start_date(config): + profiles = make_profiles() + + config["start_date"] = "2021-07-10" + stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) + assert stream.get_start_date(profiles[0], {}) == Date(2021, 7, 10) + config["start_date"] = "2021-05-10" + stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) + assert stream.get_start_date(profiles[0], {}) == Date(2021, 6, 1) + + profile_id = str(profiles[0].profileId) + stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210810"}}) == Date(2021, 8, 10) + stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210510"}}) == Date(2021, 6, 1) + + config.pop("start_date") + stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) + assert stream.get_start_date(profiles[0], {}) == Date(2021, 7, 31) + + +@freeze_time("2021-08-01 04:00:00") +def test_stream_slices_different_timezones(config): + profile1 = Profile(profileId=1, timezone="America/Los_Angeles", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) + profile2 = Profile(profileId=2, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) + stream = SponsoredProductsReportStream(config, [profile1, profile2], authenticator=mock.MagicMock()) + slices = stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state={}) + assert slices == [{"profile": profile1, "reportDate": "20210731"}, {"profile": profile2, "reportDate": "20210801"}] diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 07dfe62d9bca..cfb5c0ba2192 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -27,10 +27,10 @@ To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you 3. On the source setup page, select **Amazon Ads** from the Source type dropdown and enter a name for this connector. 4. Click `Authenticate your Amazon Ads account`. 5. Log in and Authorize to the Amazon account. -6. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)** or **Sandbox Environment**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. +6. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. 7. **Report Wait Timeout** is the maximum number of minutes the connector waits for the generation of a report for streams `Brands Reports`, `Brand Video Reports`, `Display Reports`, `Products Reports`. 8. **Report Generation Maximum Retries** is the maximum number of attempts the connector tries to generate a report for streams `Brands Reports`, `Brand Video Reports`, `Display Reports`, `Products Reports`. -9. **Start Date (Optional)** is used for generating reports starting from the specified start date. Should be in YYYY-MM-DD format and not more than 60 days in the past. If not specified today's date is used. The date for a specific profile is calculated according to its timezone, this parameter should be specified in the UTC timezone. Since it has no sense of generating report for the current day \(metrics could be changed\) it generates report for the day before \(e.g. if **Start Date** is 2021-10-11 it would use 20211010 as `reportDate` parameter for request\). +9. **Start Date (Optional)** is used for generating reports starting from the specified start date. Should be in YYYY-MM-DD format and not more than 60 days in the past. If not specified today's date is used. The date is treated in the timezone of the processed profile. 10. **Profile IDs (Optional)** you want to fetch data for. See [docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. 11. Click `Set up source`. @@ -69,7 +69,7 @@ This source is capable of syncing the following streams: ## Connector-specific features and highlights -All the reports are generated for the day before relatively to the target profile' timezone +All the reports are generated relative to the target profile' timezone. ## Performance considerations @@ -90,6 +90,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------| +| 0.1.11 | 2022-07-28 | [15031](https://github.com/airbytehq/airbyte/pull/15031) | Improve report streams date-range generation | | 0.1.10 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from schemas | | 0.1.9 | 2022-05-08 | [12541](https://github.com/airbytehq/airbyte/pull/12541) | Improve documentation for Beta | | 0.1.8 | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | From 964b2263f78db67e987c5193a3c9a487ab08926e Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 5 Aug 2022 15:24:44 +0700 Subject: [PATCH 03/25] Update Kinesis destination to use outputRecordCollector to properly store state (#15348) * Update Kinesis destination to use outputRecordCollector to properly store state * Bump version * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../seed/destination_definitions.yaml | 2 +- .../resources/seed/destination_specs.yaml | 2 +- .../connectors/destination-kinesis/Dockerfile | 2 +- .../destination-kinesis/build.gradle | 1 + .../kinesis/KinesisDestination.java | 11 +++-- .../kinesis/KinesisMessageConsumer.java | 17 +++---- .../kinesis/KinesisRecordConsumerTest.java | 47 +++++++++++++++++++ 7 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index df851ce5587f..05be34b8d059 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -140,7 +140,7 @@ - name: Kinesis destinationDefinitionId: 6d1d66d4-26ab-4602-8d32-f85894b04955 dockerRepository: airbyte/destination-kinesis - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/kinesis icon: kinesis.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index db3f3c3c91de..d07d6934f191 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2296,7 +2296,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-kinesis:0.1.3" +- dockerImage: "airbyte/destination-kinesis:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/kinesis" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-kinesis/Dockerfile b/airbyte-integrations/connectors/destination-kinesis/Dockerfile index 4513fa652bb8..05a5d4bd26d6 100644 --- a/airbyte-integrations/connectors/destination-kinesis/Dockerfile +++ b/airbyte-integrations/connectors/destination-kinesis/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-kinesis COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-kinesis diff --git a/airbyte-integrations/connectors/destination-kinesis/build.gradle b/airbyte-integrations/connectors/destination-kinesis/build.gradle index 5861e1f4a3f2..b92d73d3f8d4 100644 --- a/airbyte-integrations/connectors/destination-kinesis/build.gradle +++ b/airbyte-integrations/connectors/destination-kinesis/build.gradle @@ -24,6 +24,7 @@ dependencies { testImplementation "org.assertj:assertj-core:${assertVersion}" testImplementation "org.testcontainers:localstack:${testContainersVersion}" + testImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') diff --git a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java index 5c88b6f87bf1..2170fba76afd 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java @@ -34,7 +34,7 @@ public static void main(String[] args) throws Exception { * @return AirbyteConnectionStatus status of the connection result. */ @Override - public AirbyteConnectionStatus check(JsonNode config) { + public AirbyteConnectionStatus check(final JsonNode config) { KinesisStream kinesisStream = null; var streamName = "test_stream"; try { @@ -69,10 +69,11 @@ public AirbyteConnectionStatus check(JsonNode config) { * @return KinesisMessageConsumer for consuming Airbyte messages and streaming them to Kinesis. */ @Override - public AirbyteMessageConsumer getConsumer(JsonNode config, - ConfiguredAirbyteCatalog configuredCatalog, - Consumer outputRecordCollector) { - return new KinesisMessageConsumer(new KinesisConfig(config), configuredCatalog, outputRecordCollector); + public AirbyteMessageConsumer getConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog configuredCatalog, + final Consumer outputRecordCollector) { + final KinesisStream kinesisStream = new KinesisStream(new KinesisConfig(config)); + return new KinesisMessageConsumer(configuredCatalog, kinesisStream, outputRecordCollector); } } diff --git a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java index b76ce98c93e1..2bd0360f5a43 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java @@ -29,13 +29,11 @@ public class KinesisMessageConsumer extends FailureTrackingAirbyteMessageConsume private final Map kinesisStreams; - private AirbyteMessage lastMessage = null; - - public KinesisMessageConsumer(KinesisConfig kinesisConfig, - ConfiguredAirbyteCatalog configuredCatalog, - Consumer outputRecordCollector) { + public KinesisMessageConsumer(final ConfiguredAirbyteCatalog configuredCatalog, + final KinesisStream kinesisStream, + final Consumer outputRecordCollector) { this.outputRecordCollector = outputRecordCollector; - this.kinesisStream = new KinesisStream(kinesisConfig); + this.kinesisStream = kinesisStream; var nameTransformer = new KinesisNameTransformer(); this.kinesisStreams = configuredCatalog.getStreams().stream() .collect(Collectors.toUnmodifiableMap( @@ -60,7 +58,7 @@ protected void startTracked() { * @param message received from the Airbyte source. */ @Override - protected void acceptTracked(AirbyteMessage message) { + protected void acceptTracked(final AirbyteMessage message) { if (message.getType() == AirbyteMessage.Type.RECORD) { var messageRecord = message.getRecord(); @@ -84,7 +82,7 @@ protected void acceptTracked(AirbyteMessage message) { // throw exception and end sync? }); } else if (message.getType() == AirbyteMessage.Type.STATE) { - this.lastMessage = message; + outputRecordCollector.accept(message); } else { LOGGER.warn("Unsupported airbyte message type: {}", message.getType()); } @@ -97,13 +95,12 @@ protected void acceptTracked(AirbyteMessage message) { * @param hasFailed flag for indicating if the operation has failed. */ @Override - protected void close(boolean hasFailed) { + protected void close(final boolean hasFailed) { try { if (!hasFailed) { kinesisStream.flush(e -> { LOGGER.error("Error while streaming data to Kinesis", e); }); - this.outputRecordCollector.accept(lastMessage); } } finally { kinesisStream.close(); diff --git a/airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java b/airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java new file mode 100644 index 000000000000..a96787cce0ad --- /dev/null +++ b/airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.kinesis; + +import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.function.Consumer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@DisplayName("KinesisRecordConsumer") +@ExtendWith(MockitoExtension.class) +public class KinesisRecordConsumerTest extends PerStreamStateMessageTest { + + @Mock + private Consumer outputRecordCollector; + + @Mock + private ConfiguredAirbyteCatalog catalog; + @Mock + private KinesisStream kinesisStream; + + private KinesisMessageConsumer consumer; + + @BeforeEach + public void init() { + consumer = new KinesisMessageConsumer(catalog, kinesisStream, outputRecordCollector); + } + + @Override + protected Consumer getMockedConsumer() { + return outputRecordCollector; + } + + @Override + protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { + return consumer; + } + +} From f0e1fbfd2fe1384752a14dcf6f34221ebc262100 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 5 Aug 2022 15:39:44 +0700 Subject: [PATCH 04/25] Update Pulsar destination to use outputRecordCollector to properly store state (#15349) * Update Pulsar destination to use outputRecordCollector to properly store state * Bump version * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../seed/destination_definitions.yaml | 2 +- .../resources/seed/destination_specs.yaml | 2 +- .../connectors/destination-pulsar/Dockerfile | 2 +- .../destination-pulsar/build.gradle | 1 + .../destination/pulsar/PulsarDestination.java | 4 +- .../pulsar/PulsarRecordConsumer.java | 14 ++---- .../pulsar/PulsarRecordConsumerTest.java | 43 ++++++++++++++++--- 7 files changed, 48 insertions(+), 20 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 05be34b8d059..2cb599631b18 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -210,7 +210,7 @@ - name: Pulsar destinationDefinitionId: 2340cbba-358e-11ec-8d3d-0242ac130203 dockerRepository: airbyte/destination-pulsar - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/pulsar icon: pulsar.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index d07d6934f191..349d0913c5d8 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3479,7 +3479,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-pulsar:0.1.2" +- dockerImage: "airbyte/destination-pulsar:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/pulsar" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-pulsar/Dockerfile b/airbyte-integrations/connectors/destination-pulsar/Dockerfile index c96c68c6c3f2..1861288ae8c9 100644 --- a/airbyte-integrations/connectors/destination-pulsar/Dockerfile +++ b/airbyte-integrations/connectors/destination-pulsar/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-pulsar COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-pulsar diff --git a/airbyte-integrations/connectors/destination-pulsar/build.gradle b/airbyte-integrations/connectors/destination-pulsar/build.gradle index 4cd167744463..6e12ff7c64e5 100644 --- a/airbyte-integrations/connectors/destination-pulsar/build.gradle +++ b/airbyte-integrations/connectors/destination-pulsar/build.gradle @@ -19,6 +19,7 @@ dependencies { implementation 'org.apache.pulsar:pulsar-client:2.8.1' testImplementation libs.connectors.testcontainers.pulsar + testImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-pulsar') diff --git a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java index 960a0060deee..001423b26236 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java @@ -80,8 +80,10 @@ public AirbyteConnectionStatus check(final JsonNode config) { public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) { - return new PulsarRecordConsumer(PulsarDestinationConfig.getPulsarDestinationConfig(config), + final PulsarDestinationConfig pulsarConfig = PulsarDestinationConfig.getPulsarDestinationConfig(config); + return new PulsarRecordConsumer(pulsarConfig, catalog, + PulsarUtils.buildClient(pulsarConfig.getServiceUrl()), outputRecordCollector, namingResolver); } diff --git a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java index 851d4ea179ee..fde2db986903 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java @@ -38,10 +38,9 @@ public class PulsarRecordConsumer extends FailureTrackingAirbyteMessageConsumer private final NamingConventionTransformer nameTransformer; private final PulsarClient client; - private AirbyteMessage lastStateMessage = null; - public PulsarRecordConsumer(final PulsarDestinationConfig pulsarDestinationConfig, final ConfiguredAirbyteCatalog catalog, + final PulsarClient pulsarClient, final Consumer outputRecordCollector, final NamingConventionTransformer nameTransformer) { this.config = pulsarDestinationConfig; @@ -49,7 +48,7 @@ public PulsarRecordConsumer(final PulsarDestinationConfig pulsarDestinationConfi this.catalog = catalog; this.outputRecordCollector = outputRecordCollector; this.nameTransformer = nameTransformer; - this.client = PulsarUtils.buildClient(this.config.getServiceUrl()); + this.client = pulsarClient; } @Override @@ -60,7 +59,7 @@ protected void startTracked() { @Override protected void acceptTracked(final AirbyteMessage airbyteMessage) { if (airbyteMessage.getType() == AirbyteMessage.Type.STATE) { - lastStateMessage = airbyteMessage; + outputRecordCollector.accept(airbyteMessage); } else if (airbyteMessage.getType() == AirbyteMessage.Type.RECORD) { final AirbyteRecordMessage recordMessage = airbyteMessage.getRecord(); final Producer producer = producerMap.get(AirbyteStreamNameNamespacePair.fromRecordMessage(recordMessage)); @@ -100,9 +99,6 @@ private void sendRecord(final Producer producer, final GenericRec LOGGER.error("Error sending message to topic.", e); throw new RuntimeException("Cannot send message to Pulsar. Error: " + e.getMessage(), e); } - if (lastStateMessage != null) { - outputRecordCollector.accept(lastStateMessage); - } } } @@ -113,10 +109,6 @@ protected void close(final boolean hasFailed) { Exceptions.swallow(producer::close); }); Exceptions.swallow(client::close); - - if (lastStateMessage != null) { - outputRecordCollector.accept(lastStateMessage); - } } } diff --git a/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java b/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java index 28725a066759..7542922b8bb7 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java @@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; @@ -16,7 +15,9 @@ import com.google.common.net.InetAddresses; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.AirbyteStateMessage; @@ -41,21 +42,40 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.schema.GenericRecord; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.testcontainers.containers.PulsarContainer; import org.testcontainers.utility.DockerImageName; @DisplayName("PulsarRecordConsumer") -public class PulsarRecordConsumerTest { +@ExtendWith(MockitoExtension.class) +public class PulsarRecordConsumerTest extends PerStreamStateMessageTest { + + @Mock + private Consumer outputRecordCollector; + + private PulsarRecordConsumer consumer; + + @Mock + private PulsarDestinationConfig config; + + @Mock + private ConfiguredAirbyteCatalog catalog; + + @Mock + private PulsarClient pulsarClient; private static final StandardNameTransformer NAMING_RESOLVER = new StandardNameTransformer(); @@ -75,8 +95,8 @@ public void testBuildProducerMap(final ConfiguredAirbyteCatalog catalog, .collect(Collectors.joining(",")); final PulsarDestinationConfig config = PulsarDestinationConfig .getPulsarDestinationConfig(getConfig(brokers, topicPattern)); - - final PulsarRecordConsumer recordConsumer = new PulsarRecordConsumer(config, catalog, mock(Consumer.class), NAMING_RESOLVER); + final PulsarClient pulsarClient = PulsarUtils.buildClient(config.getServiceUrl()); + final PulsarRecordConsumer recordConsumer = new PulsarRecordConsumer(config, catalog, pulsarClient, outputRecordCollector, NAMING_RESOLVER); final Map> producerMap = recordConsumer.buildProducerMap(); assertEquals(Sets.newHashSet(catalog.getStreams()).size(), producerMap.size()); @@ -98,7 +118,8 @@ void testCannotConnectToBrokers() throws Exception { namespace, Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)))); - final PulsarRecordConsumer consumer = new PulsarRecordConsumer(config, catalog, mock(Consumer.class), NAMING_RESOLVER); + final PulsarClient pulsarClient = PulsarUtils.buildClient(config.getServiceUrl()); + final PulsarRecordConsumer consumer = new PulsarRecordConsumer(config, catalog, pulsarClient, outputRecordCollector, NAMING_RESOLVER); final List expectedRecords = getNRecords(10, streamName, namespace); assertThrows(RuntimeException.class, consumer::start); @@ -211,10 +232,22 @@ private List buildArgs(final ConfiguredAirbyteCatalog catalog, final } + @Override + protected Consumer getMockedConsumer() { + return outputRecordCollector; + } + + @Override + protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { + return consumer; + } + @BeforeEach void setup() { + // TODO: Unit tests should not use Testcontainers PULSAR = new PulsarContainer(DockerImageName.parse("apachepulsar/pulsar:2.8.1")); PULSAR.start(); + consumer = new PulsarRecordConsumer(config, catalog, pulsarClient, outputRecordCollector, NAMING_RESOLVER); } @AfterEach From ac33e193f1b7c8cf8b0d31d9d5bdfc2e5a2a618f Mon Sep 17 00:00:00 2001 From: Oleksandr Sheheda Date: Fri, 5 Aug 2022 12:24:31 +0300 Subject: [PATCH 05/25] =?UTF-8?q?=F0=9F=8E=89=20Destination=20DynamoDB:=20?= =?UTF-8?q?Handle=20per-stream=20state=20(#15350)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [15304] 🎉 Destination DynamoDB: Handle per-stream state * [15304] 🎉 Destination DynamoDB: Handle per-stream state * [15304] 🎉 Destination DynamoDB: Handle per-stream state * [15304] 🎉 Destination DynamoDB: Handle per-stream state * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../seed/destination_definitions.yaml | 2 +- .../resources/seed/destination_specs.yaml | 2 +- .../destination-dynamodb/Dockerfile | 2 +- .../destination-dynamodb/build.gradle | 2 + .../dynamodb/DynamodbConsumer.java | 8 +--- .../dynamodb/DynamodbConsumerTest.java | 48 +++++++++++++++++++ docs/integrations/destinations/dynamodb.md | 1 + 7 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 2cb599631b18..a0fbaa91712f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -80,7 +80,7 @@ - name: DynamoDB destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 dockerRepository: airbyte/destination-dynamodb - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/destinations/dynamodb icon: dynamodb.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 349d0913c5d8..59e11741de8f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1141,7 +1141,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-dynamodb:0.1.4" +- dockerImage: "airbyte/destination-dynamodb:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/dynamodb" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile index 5f80d086a636..027c8b00d737 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-dynamodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-dynamodb diff --git a/airbyte-integrations/connectors/destination-dynamodb/build.gradle b/airbyte-integrations/connectors/destination-dynamodb/build.gradle index 41bb9c7c3437..f9b3c74559ac 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/destination-dynamodb/build.gradle @@ -17,6 +17,8 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.12.47' + testImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-dynamodb') } diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java index efd8ee51f5cc..babd12b13af3 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java @@ -29,8 +29,6 @@ public class DynamodbConsumer extends FailureTrackingAirbyteMessageConsumer { private final Consumer outputRecordCollector; private final Map streamNameAndNamespaceToWriters; - private AirbyteMessage lastStateMessage = null; - public DynamodbConsumer(final DynamodbDestinationConfig dynamodbDestinationConfig, final ConfiguredAirbyteCatalog configuredCatalog, final Consumer outputRecordCollector) { @@ -80,7 +78,7 @@ protected void startTracked() throws Exception { @Override protected void acceptTracked(final AirbyteMessage airbyteMessage) throws Exception { if (airbyteMessage.getType() == AirbyteMessage.Type.STATE) { - this.lastStateMessage = airbyteMessage; + outputRecordCollector.accept(airbyteMessage); return; } else if (airbyteMessage.getType() != AirbyteMessage.Type.RECORD) { return; @@ -105,10 +103,6 @@ protected void close(final boolean hasFailed) throws Exception { for (final DynamodbWriter handler : streamNameAndNamespaceToWriters.values()) { handler.close(hasFailed); } - // DynamoDB stream uploader is all or nothing if a failure happens in the destination. - if (!hasFailed) { - outputRecordCollector.accept(lastStateMessage); - } } } diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java b/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java new file mode 100644 index 000000000000..73b544aacaea --- /dev/null +++ b/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.dynamodb; + +import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.function.Consumer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class DynamodbConsumerTest extends PerStreamStateMessageTest { + + @Mock + private Consumer outputRecordCollector; + + @InjectMocks + private DynamodbConsumer consumer; + + @Mock + private DynamodbDestinationConfig destinationConfig; + + @Mock + private ConfiguredAirbyteCatalog catalog; + + @BeforeEach + private void init() { + consumer = new DynamodbConsumer(destinationConfig, catalog, outputRecordCollector); + } + + @Override + protected Consumer getMockedConsumer() { + return outputRecordCollector; + } + + @Override + protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { + return consumer; + } + +} diff --git a/docs/integrations/destinations/dynamodb.md b/docs/integrations/destinations/dynamodb.md index 727218d51386..b02293b2bd30 100644 --- a/docs/integrations/destinations/dynamodb.md +++ b/docs/integrations/destinations/dynamodb.md @@ -58,6 +58,7 @@ This connector by default uses 10 capacity units for both Read and Write in Dyna | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-08-05 | [\#15350](https://github.com/airbytehq/airbyte/pull/15350) | Added per-stream handling | | 0.1.4 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.3 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.2 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | From a16b5d14917d4d359398540cfe7f23c2222ac1ad Mon Sep 17 00:00:00 2001 From: Oleksandr Sheheda Date: Fri, 5 Aug 2022 13:31:21 +0300 Subject: [PATCH 06/25] fixed formatting (#15359) --- .../bigquery/BigQueryRecordConsumerTest.java | 11 +- .../mongodb/MongodbRecordConsumerTest.java | 11 +- ...ryptedOracleDestinationAcceptanceTest.java | 10 +- .../oracle/OracleDestinationTest.java | 14 +- .../schemas/ad_account.json | 24 +- .../pull_request_comment_reactions.json | 1356 ++++++++--------- .../AbstractJdbcSourceAcceptanceTest.java | 6 +- .../source_okta/schemas/group_members.json | 2 +- .../src/test/resources/expected_spec.json | 8 +- .../source/oracle/OracleSource.java | 4 +- .../src/main/resources/spec.json | 8 +- .../invalid_config_oauth.json | 20 +- .../unit_tests/conftest.py | 8 +- .../unit_tests/test_source.py | 11 +- .../unit_tests/unit_test.py | 8 +- .../source/postgres/PostgresSource.java | 6 +- .../postgres/PostgresSourceOperations.java | 3 +- .../AbstractPostgresSourceDatatypeTest.java | 30 +- ...stractSshPostgresSourceAcceptanceTest.java | 12 +- .../CdcPostgresSourceAcceptanceTest.java | 16 +- .../CdcPostgresSourceDatatypeTest.java | 2 +- .../sources/PostgresSourceAcceptanceTest.java | 1 - .../postgres/CdcPostgresSourceTest.java | 6 +- .../PostgresJdbcSourceAcceptanceTest.java | 32 +- .../SnowflakeSourceOperations.java | 1 + .../SnowflakeJdbcSourceAcceptanceTest.java | 30 +- .../sample_files/state.json | 2 - 27 files changed, 830 insertions(+), 812 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java index 8e87852a506b..50fb51e4cf75 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.bigquery; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; @@ -23,11 +27,14 @@ public class BigQueryRecordConsumerTest extends PerStreamStateMessageTest { @InjectMocks private BigQueryRecordConsumer bigQueryRecordConsumer; - @Override protected Consumer getMockedConsumer() { + @Override + protected Consumer getMockedConsumer() { return outputRecordCollector; } - @Override protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { + @Override + protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { return bigQueryRecordConsumer; } + } diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java index 2b3b264a1507..5a334c6476e9 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.mongodb; import io.airbyte.db.mongodb.MongoDatabase; @@ -31,11 +35,14 @@ public class MongodbRecordConsumerTest extends PerStreamStateMessageTest { @InjectMocks private MongodbRecordConsumer mongodbRecordConsumer; - @Override protected Consumer getMockedConsumer() { + @Override + protected Consumer getMockedConsumer() { return outputRecordCollector; } - @Override protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { + @Override + protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { return mongodbRecordConsumer; } + } diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index 5dadbd5a69c6..fd8a61a9907b 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -67,9 +67,9 @@ protected JsonNode getConfig() { @Override protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) + final String streamName, + final String namespace, + final JsonNode streamSchema) throws Exception { return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) .stream() @@ -110,8 +110,8 @@ protected boolean supportObjectDataTypeTest() { @Override protected List retrieveNormalizedRecords(final TestDestinationEnv env, - final String streamName, - final String namespace) + final String streamName, + final String namespace) throws Exception { final String tableName = namingResolver.getIdentifier(streamName); return retrieveRecordsFromTable(tableName, namespace); diff --git a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java index fff62dc01973..8a0b82141073 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java @@ -130,13 +130,13 @@ void testExtraParams() { private JsonNode buildConfigWithExtraJdbcParameters(String extraParam) { return Jsons.jsonNode(com.google.common.collect.ImmutableMap.of( - "host", "localhost", - "port", 1773, - "sid", "ORCL", - "database", "db", - "username", "username", - "password", "verysecure", - "jdbc_url_params", extraParam)); + "host", "localhost", + "port", 1773, + "sid", "ORCL", + "database", "db", + "username", "username", + "password", "verysecure", + "jdbc_url_params", extraParam)); } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json index 59faf7e23b10..df1b955216a7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json @@ -79,20 +79,20 @@ }, "failed_delivery_checks": { "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "summary": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "check_name": { - "type": ["null", "string"] - } + "items": { + "type": ["null", "object"], + "properties": { + "summary": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "check_name": { + "type": ["null", "string"] } } + } }, "fb_entity": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-github/unit_tests/pull_request_comment_reactions.json b/airbyte-integrations/connectors/source-github/unit_tests/pull_request_comment_reactions.json index f7d95aee8b60..ca8547fc4934 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/pull_request_comment_reactions.json +++ b/airbyte-integrations/connectors/source-github/unit_tests/pull_request_comment_reactions.json @@ -1,744 +1,744 @@ [ - { - "data": { - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" + { + "data": { + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + }, + "pullRequests": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "endCursor" + }, + "totalCount": 4, + "nodes": [ + { + "node_id": "pull_request1", + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" }, - "pullRequests": { - "pageInfo": { + "totalCount": 2, + "nodes": [ + { + "node_id": "review1", + "comments": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 2, + "nodes": [ + { + "node_id": "comment1", + "id": "comment1", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 2, + "nodes": [ + { + "node_id": "reaction1", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + }, + { + "node_id": "reaction2", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } + }, + { + "node_id": "comment2", + "id": "comment2", + "reactions": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "endCursor" + }, + "totalCount": 3, + "nodes": [ + { + "node_id": "reaction3", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + }, + { + "node_id": "reaction4", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } + } + ] + } + }, + { + "node_id": "review2", + "comments": { + "pageInfo": { "hasNextPage": true, "endCursor": "endCursor" - }, - "totalCount": 4, - "nodes": [ + }, + "totalCount": 3, + "nodes": [ { - "node_id": "pull_request1", - "reviews": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "review1", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "comment1", - "id": "comment1", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "reaction1", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - }, - { - "node_id": "reaction2", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - }, - { - "node_id": "comment2", - "id": "comment2", - "reactions": { - "pageInfo": { - "hasNextPage": true, - "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ - { - "node_id": "reaction3", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - }, - { - "node_id": "reaction4", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - } - ] - } - }, - { - "node_id": "review2", - "comments": { - "pageInfo": { - "hasNextPage": true, - "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ - { - "node_id": "comment3", - "id": "comment3", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - }, - { - "node_id": "comment4", - "id": "comment4", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - } - ] - } - } - ] - } + "node_id": "comment3", + "id": "comment3", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 0, + "nodes": [] + } }, { - "node_id": "pull_request2", - "reviews": { - "pageInfo": { - "hasNextPage": true, - "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ - { - "node_id": "review3", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - }, - { - "node_id": "review4", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - } - ] - } + "node_id": "comment4", + "id": "comment4", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 0, + "nodes": [] + } } - ] - } - } - } - }, - { - "data": { - "node": { - "__typename": "PullRequestReviewComment", - "node_id": "comment2", - "id": "comment2", - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" + ] } + } + ] + } + }, + { + "node_id": "pull_request2", + "reviews": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "endCursor" }, - "reactions": { - "pageInfo": { + "totalCount": 3, + "nodes": [ + { + "node_id": "review3", + "comments": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ - { - "node_id": "reaction5", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } + }, + "totalCount": 0, + "nodes": [] + } + }, + { + "node_id": "review4", + "comments": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 0, + "nodes": [] + } + } + ] + } } + ] } - }, - { - "data": { - "node": { - "__typename": "PullRequestReview", - "node_id": "review2", - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" - } + } + } + }, + { + "data": { + "node": { + "__typename": "PullRequestReviewComment", + "node_id": "comment2", + "id": "comment2", + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + } + }, + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 3, + "nodes": [ + { + "node_id": "reaction5", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } + } + } + }, + { + "data": { + "node": { + "__typename": "PullRequestReview", + "node_id": "review2", + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + } + }, + "comments": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 3, + "nodes": [ + { + "node_id": "comment5", + "id": "comment5", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" }, - "comments": { - "pageInfo": { + "totalCount": 1, + "nodes": [ + { + "node_id": "reaction6", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } + } + ] + } + } + } + }, + { + "data": { + "node": { + "__typename": "PullRequest", + "node_id": "pull_request2", + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + } + }, + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 3, + "nodes": [ + { + "node_id": "review5", + "comments": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 2, + "nodes": [ + { + "node_id": "comment6", + "id": "comment6", + "reactions": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ + }, + "totalCount": 0, + "nodes": [] + } + }, + { + "node_id": "comment7", + "id": "comment7", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 1, + "nodes": [ { - "node_id": "comment5", - "id": "comment5", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "reaction6", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } + "node_id": "reaction7", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" } - ] - } + ] + } + } + ] + } } + ] } - }, - { - "data": { - "node": { - "__typename": "PullRequest", - "node_id": "pull_request2", - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" - } + } + } + }, + { + "data": { + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + }, + "pullRequests": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 4, + "nodes": [ + { + "node_id": "pull_request3", + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" }, - "reviews": { - "pageInfo": { + "totalCount": 0, + "nodes": [] + } + }, + { + "node_id": "pull_request4", + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 1, + "nodes": [ + { + "node_id": "review6", + "comments": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ + }, + "totalCount": 1, + "nodes": [ { - "node_id": "review5", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "comment6", - "id": "comment6", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - }, - { - "node_id": "comment7", - "id": "comment7", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "reaction7", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - } - ] - } + "node_id": "comment8", + "id": "comment8", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 1, + "nodes": [ + { + "node_id": "reaction8", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } } - ] - } + ] + } + } + ] + } } + ] } - }, - { - "data": { - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" + } + } + }, + { + "data": { + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + }, + "pullRequests": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "endCursor" + }, + "totalCount": 4, + "nodes": [ + { + "node_id": "pull_request1", + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" }, - "pullRequests": { - "pageInfo": { + "totalCount": 2, + "nodes": [ + { + "node_id": "review1", + "comments": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 4, - "nodes": [ + }, + "totalCount": 2, + "nodes": [ { - "node_id": "pull_request3", - "reviews": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } + "node_id": "comment1", + "id": "comment1", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 2, + "nodes": [ + { + "node_id": "reaction1", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + }, + { + "node_id": "reaction2", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } }, { - "node_id": "pull_request4", - "reviews": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "review6", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "comment8", - "id": "comment8", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "reaction8", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - } - ] - } - } - ] - } + "node_id": "comment2", + "id": "comment2", + "reactions": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "endCursor" + }, + "totalCount": 4, + "nodes": [ + { + "node_id": "reaction3", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + }, + { + "node_id": "reaction4", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } } - ] - } - } - } - }, - { - "data": { - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" - }, - "pullRequests": { - "pageInfo": { + ] + } + }, + { + "node_id": "review2", + "comments": { + "pageInfo": { "hasNextPage": true, "endCursor": "endCursor" - }, - "totalCount": 4, - "nodes": [ + }, + "totalCount": 3, + "nodes": [ { - "node_id": "pull_request1", - "reviews": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "review1", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "comment1", - "id": "comment1", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "reaction1", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - }, - { - "node_id": "reaction2", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - }, - { - "node_id": "comment2", - "id": "comment2", - "reactions": { - "pageInfo": { - "hasNextPage": true, - "endCursor": "endCursor" - }, - "totalCount": 4, - "nodes": [ - { - "node_id": "reaction3", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - }, - { - "node_id": "reaction4", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - } - ] - } - }, - { - "node_id": "review2", - "comments": { - "pageInfo": { - "hasNextPage": true, - "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ - { - "node_id": "comment3", - "id": "comment3", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - }, - { - "node_id": "comment4", - "id": "comment4", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - } - ] - } - } - ] - } + "node_id": "comment3", + "id": "comment3", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 0, + "nodes": [] + } }, { - "node_id": "pull_request2", - "reviews": { - "pageInfo": { - "hasNextPage": true, - "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ - { - "node_id": "review3", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - }, - { - "node_id": "review4", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - } - ] - } + "node_id": "comment4", + "id": "comment4", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 0, + "nodes": [] + } } - ] - } - } - } - }, - { - "data": { - "node": { - "__typename": "PullRequestReviewComment", - "node_id": "comment2", - "id": "comment2", - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" + ] } + } + ] + } + }, + { + "node_id": "pull_request2", + "reviews": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "endCursor" }, - "reactions": { - "pageInfo": { + "totalCount": 3, + "nodes": [ + { + "node_id": "review3", + "comments": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 4, - "nodes": [ - { - "node_id": "reaction5", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - }, - { - "node_id": "reaction9", - "user": {}, - "created_at": "2022-01-02T00:00:01Z" - } - ] - } - } - } - }, - { - "data": { - "node": { - "__typename": "PullRequestReview", - "node_id": "review2", - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" + }, + "totalCount": 0, + "nodes": [] } - }, - "comments": { - "pageInfo": { + }, + { + "node_id": "review4", + "comments": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ - { - "node_id": "comment5", - "id": "comment5", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "reaction6", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - } - ] - } + }, + "totalCount": 0, + "nodes": [] + } + } + ] + } } + ] } - }, - { - "data": { - "node": { - "__typename": "PullRequest", - "node_id": "pull_request2", - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" - } + } + } + }, + { + "data": { + "node": { + "__typename": "PullRequestReviewComment", + "node_id": "comment2", + "id": "comment2", + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + } + }, + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 4, + "nodes": [ + { + "node_id": "reaction5", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + }, + { + "node_id": "reaction9", + "user": {}, + "created_at": "2022-01-02T00:00:01Z" + } + ] + } + } + } + }, + { + "data": { + "node": { + "__typename": "PullRequestReview", + "node_id": "review2", + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + } + }, + "comments": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 3, + "nodes": [ + { + "node_id": "comment5", + "id": "comment5", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 1, + "nodes": [ + { + "node_id": "reaction6", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + } + ] + } + } + ] + } + } + } + }, + { + "data": { + "node": { + "__typename": "PullRequest", + "node_id": "pull_request2", + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + } + }, + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 3, + "nodes": [ + { + "node_id": "review5", + "comments": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" }, - "reviews": { - "pageInfo": { + "totalCount": 2, + "nodes": [ + { + "node_id": "comment6", + "id": "comment6", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 0, + "nodes": [] + } + }, + { + "node_id": "comment7", + "id": "comment7", + "reactions": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 3, - "nodes": [ + }, + "totalCount": 1, + "nodes": [ { - "node_id": "review5", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "comment6", - "id": "comment6", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - }, - { - "node_id": "comment7", - "id": "comment7", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "reaction7", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - } - ] - } - } - ] - } + "node_id": "reaction7", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" } - ] - } + ] + } + } + ] + } } + ] } - }, - { - "data": { - "repository": { - "name": "airbyte", - "owner": { - "login": "airbytehq" + } + } + }, + { + "data": { + "repository": { + "name": "airbyte", + "owner": { + "login": "airbytehq" + }, + "pullRequests": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 4, + "nodes": [ + { + "node_id": "pull_request3", + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" }, - "pullRequests": { - "pageInfo": { + "totalCount": 0, + "nodes": [] + } + }, + { + "node_id": "pull_request4", + "reviews": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 1, + "nodes": [ + { + "node_id": "review6", + "comments": { + "pageInfo": { "hasNextPage": false, "endCursor": "endCursor" - }, - "totalCount": 4, - "nodes": [ - { - "node_id": "pull_request3", - "reviews": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 0, - "nodes": [] - } - }, + }, + "totalCount": 1, + "nodes": [ { - "node_id": "pull_request4", - "reviews": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "review6", - "comments": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 1, - "nodes": [ - { - "node_id": "comment8", - "id": "comment8", - "reactions": { - "pageInfo": { - "hasNextPage": false, - "endCursor": "endCursor" - }, - "totalCount": 2, - "nodes": [ - { - "node_id": "reaction8", - "user": {}, - "created_at": "2022-01-01T00:00:01Z" - }, - { - "node_id": "reaction10", - "user": {}, - "created_at": "2022-01-02T00:00:01Z" - } - ] - } - } - ] - } - } - ] - } + "node_id": "comment8", + "id": "comment8", + "reactions": { + "pageInfo": { + "hasNextPage": false, + "endCursor": "endCursor" + }, + "totalCount": 2, + "nodes": [ + { + "node_id": "reaction8", + "user": {}, + "created_at": "2022-01-01T00:00:01Z" + }, + { + "node_id": "reaction10", + "user": {}, + "created_at": "2022-01-02T00:00:01Z" + } + ] + } } - ] - } + ] + } + } + ] + } } + ] } + } } + } ] diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java index 1e5eea3bbcaf..69eb854053b0 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java @@ -20,16 +20,15 @@ import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.AirbyteGlobalState; import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.test.utils.PostgreSQLContainerHelper; -import io.airbyte.integrations.util.HostPortResolver; import java.sql.JDBCType; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -188,8 +187,7 @@ void testCustomParametersOverwriteDefaultParametersExpectException() { final Map customParameters = JdbcUtils.parseJdbcParameters(config, JdbcUtils.CONNECTION_PROPERTIES_KEY, "&"); final Map defaultParameters = Map.of( "ssl", "true", - "sslmode", "require" - ); + "sslmode", "require"); assertThrows(IllegalArgumentException.class, () -> { assertCustomParametersDontOverwriteDefaultParameters(customParameters, defaultParameters); }); diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json index e3e9d234fc1f..c96ece1029ea 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json @@ -209,4 +209,4 @@ "type": ["null", "object"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json index 17e68834058f..07d2e50b1b0b 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json @@ -30,9 +30,7 @@ { "title": "Service name", "description": "Use service name", - "required": [ - "service_name" - ], + "required": ["service_name"], "properties": { "connection_type": { "type": "string", @@ -50,9 +48,7 @@ { "title": "System ID (SID)", "description": "Use SID (Oracle System Identifier)", - "required": [ - "sid" - ], + "required": ["sid"], "properties": { "connection_type": { "type": "string", diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index d0b084d45ce1..68f3e7f3a4d0 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -81,7 +81,8 @@ public JsonNode toDatabaseConfig(final JsonNode config) { final String connectionType = connectionData.has("connection_type") ? connectionData.get("connection_type").asText() : UNRECOGNIZED; connectionString = switch (connectionType) { - case SERVICE_NAME -> buildConnectionString(config, protocol.toString(), SERVICE_NAME.toUpperCase(), config.get(CONNECTION_DATA).get(SERVICE_NAME).asText()); + case SERVICE_NAME -> buildConnectionString(config, protocol.toString(), SERVICE_NAME.toUpperCase(), + config.get(CONNECTION_DATA).get(SERVICE_NAME).asText()); case SID -> buildConnectionString(config, protocol.toString(), SID.toUpperCase(), config.get(CONNECTION_DATA).get(SID).asText()); default -> throw new IllegalArgumentException("Unrecognized connection type: " + connectionType); }; @@ -212,4 +213,5 @@ private String buildConnectionString(JsonNode config, String protocol, String co connectionTypeName, connectionTypeValue); } + } diff --git a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json index a0fbd7624dc9..3dce50d5be3a 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json @@ -30,9 +30,7 @@ { "title": "Service name", "description": "Use service name", - "required": [ - "service_name" - ], + "required": ["service_name"], "properties": { "connection_type": { "type": "string", @@ -50,9 +48,7 @@ { "title": "System ID (SID)", "description": "Use SID (Oracle System Identifier)", - "required": [ - "sid" - ], + "required": ["sid"], "properties": { "connection_type": { "type": "string", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json index 8e3feb51bc11..21019786e6a3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json @@ -1,11 +1,11 @@ { - "credentials": { - "auth_type": "oauth2.0", - "client_id": "AWA__", - "secret": "ENC__", - "refresh_token": "__" - }, - "start_date": "2021-07-03T00:00:00+00:00", - "end_date": "2021-07-04T23:59:59+00:00", - "is_sandbox": false -} \ No newline at end of file + "credentials": { + "auth_type": "oauth2.0", + "client_id": "AWA__", + "secret": "ENC__", + "refresh_token": "__" + }, + "start_date": "2021-07-03T00:00:00+00:00", + "end_date": "2021-07-04T23:59:59+00:00", + "is_sandbox": false +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py index cbdbcd13702b..3d3f66920551 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py @@ -43,7 +43,7 @@ def prod_config(): "secret": "some_secret", "start_date": "2021-07-01T00:00:00+00:00", "end_date": "2021-07-10T00:00:00+00:00", - "is_sandbox": False + "is_sandbox": False, } @@ -57,7 +57,7 @@ def sandbox_config(): "secret": "some_secret", "start_date": "2021-07-01T00:00:00+00:00", "end_date": "2021-07-10T00:00:00+00:00", - "is_sandbox": True + "is_sandbox": True, } @@ -71,11 +71,11 @@ def new_prod_config(): "auth_type": "oauth2.0", "client_id": "some_client_id", "client_secret": "some_client_secret", - "refresh_token": "some_refresh_token" + "refresh_token": "some_refresh_token", }, "start_date": "2021-07-01T00:00:00+00:00", "end_date": "2021-07-10T00:00:00+00:00", - "is_sandbox": False + "is_sandbox": False, } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_source.py index 6d84ed2c59e2..43f7afdea63e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_source.py @@ -8,7 +8,6 @@ class TestAuthentication: - def test_init_token_authentication_init(self, authenticator_instance): assert isinstance(authenticator_instance, PayPalOauth2Authenticator) @@ -38,13 +37,19 @@ def test_streams_count(self, prod_config, source_instance): def test_check_connection_ok(self, requests_mock, prod_config, api_endpoint, transactions, source_instance): requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - url = f'{api_endpoint}/v1/reporting/transactions' + '?start_date=2021-07-01T00%3A00%3A00%2B00%3A00&end_date=2021-07-02T00%3A00%3A00%2B00%3A00&fields=all&page_size=500&page=1' + url = ( + f"{api_endpoint}/v1/reporting/transactions" + + "?start_date=2021-07-01T00%3A00%3A00%2B00%3A00&end_date=2021-07-02T00%3A00%3A00%2B00%3A00&fields=all&page_size=500&page=1" + ) requests_mock.get(url, json=transactions) assert source_instance.check_connection(logger=MagicMock(), config=prod_config) == (True, None) def test_check_connection_error(self, requests_mock, prod_config, api_endpoint, source_instance): requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - url = f'{api_endpoint}/v1/reporting/transactions' + '?start_date=2021-07-01T00%3A00%3A00%2B00%3A00&end_date=2021-07-02T00%3A00%3A00%2B00%3A00&fields=all&page_size=500&page=1' + url = ( + f"{api_endpoint}/v1/reporting/transactions" + + "?start_date=2021-07-01T00%3A00%3A00%2B00%3A00&end_date=2021-07-02T00%3A00%3A00%2B00%3A00&fields=all&page_size=500&page=1" + ) requests_mock.get(url, status_code=400, json={}) assert not source_instance.check_connection(logger=MagicMock(), config=prod_config)[0] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index d6d7bc268335..c031530a7c37 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -23,7 +23,11 @@ def test_transactions_transform_function(): transformer = stream.transformer input_data = {"transaction_amount": "123.45", "transaction_id": "111", "transaction_status": "done"} schema = stream.get_json_schema() - schema['properties'] = {"transaction_amount": {"type": "number"}, "transaction_id": {"type": "integer"}, "transaction_status": {"type": "string"}} + schema["properties"] = { + "transaction_amount": {"type": "number"}, + "transaction_id": {"type": "integer"}, + "transaction_status": {"type": "string"}, + } transformer.transform(input_data, schema) expected_data = {"transaction_amount": 123.45, "transaction_id": 111, "transaction_status": "done"} assert input_data == expected_data @@ -327,7 +331,7 @@ def test_unnest_field(): def test_get_last_refreshed_datetime(requests_mock, prod_config, api_endpoint): stream = Balances(authenticator=NoAuth(), **prod_config) requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - url = f'{api_endpoint}/v1/reporting/balances' + '?as_of_time=2021-07-01T00%3A00%3A00%2B00%3A00' + url = f"{api_endpoint}/v1/reporting/balances" + "?as_of_time=2021-07-01T00%3A00%3A00%2B00%3A00" requests_mock.get(url, json={}) assert not stream.get_last_refreshed_datetime(SyncMode.full_refresh) diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index ed18a3cfed45..5cbe4600bf8a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -306,8 +306,10 @@ public List> getIncrementalIterators(final Jsons.clone(PostgresCdcProperties.getDebeziumDefaultProperties(sourceConfig)), catalog, state, - // We can assume that there will be only 1 replication slot cause before the sync starts for Postgres CDC, - // we run all the check operations and one of the check validates that the replication slot exists and has only 1 entry + // We can assume that there will be only 1 replication slot cause before the sync starts for + // Postgres CDC, + // we run all the check operations and one of the check validates that the replication slot exists + // and has only 1 entry getReplicationSlot(database, sourceConfig).get(0), sourceConfig); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index 8e9be8666cee..03a7a6d910e1 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -216,7 +216,8 @@ public void setJsonField(final ResultSet resultSet, final int colIndex, final Ob protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { LocalDate date = getObject(resultSet, index, LocalDate.class); if (isBce(date)) { - // java.time uses a year 0, but the standard AD/BC system does not. So we just subtract one to hack around this difference. + // java.time uses a year 0, but the standard AD/BC system does not. So we just subtract one to hack + // around this difference. date = date.minusYears(1); } node.put(columnName, resolveEra(date, date.toString())); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java index 3f85a5f3c637..afe2854743aa 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java @@ -176,13 +176,13 @@ protected void initTests() { // Debezium does not handle era indicators (AD nd BC) // https://github.com/airbytehq/airbyte/issues/14590 - addDataTypeTestData( - TestDataHolder.builder() - .sourceType("date") - .airbyteType(JsonSchemaType.STRING_DATE) - .addInsertValues("'1999-01-08'", "'1991-02-10 BC'", "null") - .addExpectedValues("1999-01-08", "1991-02-10 BC", null) - .build()); + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("date") + .airbyteType(JsonSchemaType.STRING_DATE) + .addInsertValues("'1999-01-08'", "'1991-02-10 BC'", "null") + .addExpectedValues("1999-01-08", "1991-02-10 BC", null) + .build()); for (final String type : Set.of("double precision", "float", "float8")) { addDataTypeTestData( @@ -283,17 +283,17 @@ protected void initTests() { "null", "'999.99'", "'1,001.01'", "'-1,000'", "'$999.99'", "'$1001.01'", "'-$1,000'" - // max values for Money type: "-92233720368547758.08", "92233720368547758.07" - // Debezium has wrong parsing for values more than 999999999999999 and less than -999999999999999 - // https://github.com/airbytehq/airbyte/issues/7338 - /*"'-92233720368547758.08'", "'92233720368547758.07'"*/) + // max values for Money type: "-92233720368547758.08", "92233720368547758.07" + // Debezium has wrong parsing for values more than 999999999999999 and less than -999999999999999 + // https://github.com/airbytehq/airbyte/issues/7338 + /* "'-92233720368547758.08'", "'92233720368547758.07'" */) .addExpectedValues( null, // Double#toString method is necessary here because sometimes the output // has unexpected decimals, e.g. Double.toString(-1000) is -1000.0 "999.99", "1001.01", Double.toString(-1000), "999.99", "1001.01", Double.toString(-1000) - /*"-92233720368547758.08", "92233720368547758.07"*/) + /* "-92233720368547758.08", "92233720368547758.07" */) .build()); // Blocked by https://github.com/airbytehq/airbyte/issues/8902 @@ -423,7 +423,8 @@ protected void initTests() { .addInsertValues( "TIMESTAMP '2004-10-19 10:23:00'", "TIMESTAMP '2004-10-19 10:23:54.123456'", - // A random BCE date. Old enough that converting it to/from an Instant results in discrepancies from inconsistent leap year handling + // A random BCE date. Old enough that converting it to/from an Instant results in discrepancies from + // inconsistent leap year handling "TIMESTAMP '3004-10-19 10:23:54.123456 BC'", // The earliest possible timestamp in CE "TIMESTAMP '0001-01-01 00:00:00.000000'", @@ -451,7 +452,8 @@ protected void initTests() { // 10:23-08 == 18:23Z "TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08'", "TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08'", - // A random BCE date. Old enough that converting it to/from an Instant results in discrepancies from inconsistent leap year handling + // A random BCE date. Old enough that converting it to/from an Instant results in discrepancies from + // inconsistent leap year handling "TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC'", // The earliest possible timestamp in CE (16:00-08 == 00:00Z) "TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC'", diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java index 2e34b5e0629d..58e596e07965 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java @@ -118,18 +118,18 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME, - Field.of("id", JsonSchemaType.INTEGER), - Field.of("name", JsonSchemaType.STRING)) + STREAM_NAME, + Field.of("id", JsonSchemaType.INTEGER), + Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), new ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME2, - Field.of("id", JsonSchemaType.INTEGER), - Field.of("name", JsonSchemaType.STRING)) + STREAM_NAME2, + Field.of("id", JsonSchemaType.INTEGER), + Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index 92a618655098..41b358ec562e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -143,20 +143,20 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME, - NAMESPACE, - Field.of("id", JsonSchemaType.INTEGER), - Field.of("name", JsonSchemaType.STRING)) + STREAM_NAME, + NAMESPACE, + Field.of("id", JsonSchemaType.INTEGER), + Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), new ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME2, - NAMESPACE, - Field.of("id", JsonSchemaType.INTEGER), - Field.of("name", JsonSchemaType.STRING)) + STREAM_NAME2, + NAMESPACE, + Field.of("id", JsonSchemaType.INTEGER), + Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java index c609f07cec77..37abb3a3c544 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java @@ -13,7 +13,6 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.integrations.util.HostPortResolver; -import io.airbyte.protocol.models.JsonSchemaType; import java.util.List; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; @@ -98,4 +97,5 @@ protected void tearDown(final TestDestinationEnv testEnv) { public boolean testCatalog() { return true; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index ca5e44db3bf2..c72bb857dd44 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -13,7 +13,6 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.AbstractSourceConnectorTest; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.integrations.util.HostPortResolver; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 68f34b6df5a9..f463f29d7c50 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -66,7 +66,6 @@ import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; @@ -110,7 +109,7 @@ protected void setup() throws SQLException { PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), container); config = getConfig(dbName); - fullReplicationSlot = SLOT_NAME_BASE + "_" + dbName; + fullReplicationSlot = SLOT_NAME_BASE + "_" + dbName; dslContext = getDslContext(config); database = getDatabase(dslContext); super.setup(); @@ -553,7 +552,8 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { writeModelRecord(record); } - // Triggering sync with the first sync's state only which would mimic a scenario that the second sync failed on destination end and we didn't save state + // Triggering sync with the first sync's state only which would mimic a scenario that the second + // sync failed on destination end and we didn't save state final AutoCloseableIterator thirdBatchIterator = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java index bb8f6b715bfb..952e3690d7fd 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java @@ -337,25 +337,25 @@ protected void executeStatementReadIncrementallyTwice() throws SQLException { protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(Lists.newArrayList( CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE), - Field.of(COL_WAKEUP_AT, JsonSchemaType.STRING_TIME_WITH_TIMEZONE), - Field.of(COL_LAST_VISITED_AT, JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE), - Field.of(COL_LAST_COMMENT_AT, JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE)) + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE), + Field.of(COL_WAKEUP_AT, JsonSchemaType.STRING_TIME_WITH_TIMEZONE), + Field.of(COL_LAST_VISITED_AT, JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE), + Field.of(COL_LAST_COMMENT_AT, JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE), - Field.of(COL_WAKEUP_AT, JsonSchemaType.STRING_TIME_WITH_TIMEZONE), - Field.of(COL_LAST_VISITED_AT, JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE), - Field.of(COL_LAST_COMMENT_AT, JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE)) + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE), + Field.of(COL_WAKEUP_AT, JsonSchemaType.STRING_TIME_WITH_TIMEZONE), + Field.of(COL_LAST_VISITED_AT, JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE), + Field.of(COL_LAST_COMMENT_AT, JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(Collections.emptyList()), CatalogHelpers.createAirbyteStream( diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java index c7d5d3636972..4121c0c5363b 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -58,4 +58,5 @@ public JsonSchemaType getJsonType(JDBCType jdbcType) { default -> JsonSchemaType.STRING; }; } + } diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java index b9be72d1818a..5e64941eb7e9 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java @@ -103,27 +103,27 @@ void testCheckFailure() throws Exception { protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.NUMBER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.NUMBER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.NUMBER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.NUMBER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(Collections.emptyList()), CatalogHelpers.createAirbyteStream( - TABLE_NAME_COMPOSITE_PK, - defaultNamespace, - Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), - Field.of(COL_LAST_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME_COMPOSITE_PK, + defaultNamespace, + Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), + Field.of(COL_LAST_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey( List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); diff --git a/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json b/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json index 68263ca6b0fc..b934e6944f43 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json +++ b/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json @@ -43,6 +43,4 @@ "date_modified": "2021-06-10T11:06:43+00:00" } } - - } From 2344b460d3d082e6340154cdb3654eedd6d497b1 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Fri, 5 Aug 2022 14:34:20 +0300 Subject: [PATCH 07/25] Source Hubspot: fix Deals stream schema (#15354) * #393 oncall. Source Hubspot: fix Deals stream schema * #393 source hubspot: upd changelog * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../main/resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-hubspot/Dockerfile | 2 +- .../source_hubspot/schemas/deals.json | 17 ----------------- docs/integrations/sources/hubspot.md | 1 + 5 files changed, 4 insertions(+), 20 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9be4fb2490f0..a285e9a4bc2e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -414,7 +414,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.80 + dockerImageTag: 0.1.81 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 65e484cebc24..71a2a52ee125 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3709,7 +3709,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.1.80" +- dockerImage: "airbyte/source-hubspot:0.1.81" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 7cc9b1d4576e..27daf07d5e16 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.80 +LABEL io.airbyte.version=0.1.81 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json index 9049c053c6fc..b1ed1e7d874b 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json @@ -5,23 +5,6 @@ "id": { "type": ["null", "string"] }, - "dealstage": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "integer"] - }, - "versions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"] - } - } - } - }, "properties": { "type": ["null", "object"], "properties": { diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 3387ec58a6d3..93e911b2d8a5 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -129,6 +129,7 @@ Now that you have set up the HubSpot source connector, check out the following H | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.81 | 2022-08-05 | [15354](https://github.com/airbytehq/airbyte/pull/15354) | Fix `Deals` stream schema | | 0.1.80 | 2022-08-01 | [15156](https://github.com/airbytehq/airbyte/pull/15156) | Fix 401 error while retrieving associations using OAuth | | 0.1.79 | 2022-07-28 | [15144](https://github.com/airbytehq/airbyte/pull/15144) | Revert v0.1.78 due to permission issues | | 0.1.78 | 2022-07-28 | [15099](https://github.com/airbytehq/airbyte/pull/15099) | Fix to fetch associations when using incremental mode | From 2afa5d026dd9cc44d1c1cca4b53e29cd5f4275d5 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 5 Aug 2022 18:40:20 +0700 Subject: [PATCH 08/25] 15308 Destination PubSub: Handle per-stream state (#15355) * 15308 Destination PubSub: Handle per-stream state * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III Co-authored-by: Oleksandr Sheheda --- .../seed/destination_definitions.yaml | 2 +- .../resources/seed/destination_specs.yaml | 2 +- .../connectors/destination-pubsub/Dockerfile | 2 +- .../destination-pubsub/build.gradle | 2 + .../destination/pubsub/PubsubConsumer.java | 6 +-- .../pubsub/PubsubConsumerTest.java | 47 +++++++++++++++++++ 6 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index a0fbaa91712f..ca13c26351f0 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -126,7 +126,7 @@ - name: Google PubSub destinationDefinitionId: 356668e2-7e34-47f3-a3b0-67a8a481b692 dockerRepository: airbyte/destination-pubsub - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/destinations/pubsub icon: googlepubsub.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 59e11741de8f..fb89aed4be8a 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1972,7 +1972,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-pubsub:0.1.5" +- dockerImage: "airbyte/destination-pubsub:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/pubsub" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-pubsub/Dockerfile b/airbyte-integrations/connectors/destination-pubsub/Dockerfile index 4bd1e25450c1..283e66285e67 100644 --- a/airbyte-integrations/connectors/destination-pubsub/Dockerfile +++ b/airbyte-integrations/connectors/destination-pubsub/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-pubsub COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-pubsub diff --git a/airbyte-integrations/connectors/destination-pubsub/build.gradle b/airbyte-integrations/connectors/destination-pubsub/build.gradle index 6f6ecad1a8e2..14dc95279e67 100644 --- a/airbyte-integrations/connectors/destination-pubsub/build.gradle +++ b/airbyte-integrations/connectors/destination-pubsub/build.gradle @@ -17,6 +17,8 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + testImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-pubsub') } diff --git a/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java b/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java index cef941529bf2..38f78ce0a7e3 100644 --- a/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java +++ b/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java @@ -39,7 +39,6 @@ public class PubsubConsumer extends FailureTrackingAirbyteMessageConsumer { private final Consumer outputRecordCollector; private final Map> attributes; private Publisher publisher; - private AirbyteMessage lastStateMessage; public PubsubConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, @@ -47,7 +46,6 @@ public PubsubConsumer(final JsonNode config, this.outputRecordCollector = outputRecordCollector; this.config = config; this.catalog = catalog; - this.lastStateMessage = null; this.attributes = Maps.newHashMap(); this.publisher = null; LOGGER.info("initializing consumer."); @@ -82,8 +80,7 @@ protected void startTracked() throws Exception { @Override protected void acceptTracked(final AirbyteMessage msg) throws Exception { if (msg.getType() == Type.STATE) { - lastStateMessage = msg; - outputRecordCollector.accept(lastStateMessage); + outputRecordCollector.accept(msg); return; } else if (msg.getType() != Type.RECORD) { return; @@ -114,7 +111,6 @@ protected void close(final boolean hasFailed) throws Exception { if (!hasFailed) { publisher.shutdown(); LOGGER.info("shutting down consumer."); - outputRecordCollector.accept(lastStateMessage); } } diff --git a/airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java b/airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java new file mode 100644 index 000000000000..34ef00718d02 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integration.destination.pubsub; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.integrations.destination.pubsub.PubsubConsumer; +import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.function.Consumer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class PubsubConsumerTest extends PerStreamStateMessageTest { + + @Mock + private Consumer outputRecordCollector; + + private PubsubConsumer consumer; + + @Mock + private JsonNode config; + @Mock + private ConfiguredAirbyteCatalog catalog; + + @BeforeEach + public void init() { + consumer = new PubsubConsumer(config, catalog, outputRecordCollector); + } + + @Override + protected Consumer getMockedConsumer() { + return outputRecordCollector; + } + + @Override + protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { + return consumer; + } + +} From 36319d70b21f30578538bc66745776fd81700669 Mon Sep 17 00:00:00 2001 From: Eugene Date: Fri, 5 Aug 2022 15:14:20 +0300 Subject: [PATCH 09/25] [15245] Destination-mysql: fixed normalization tests after changes in python part (#15362) Destination-mysql: fixed normalization tests after changes in python part --- .../AdvancedTestDataComparator.java | 1 + .../mysql/MySqlTestDataComparator.java | 33 ++++++++++++++++++- .../postgres/PostgresTestDataComparator.java | 1 - 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java index dd775e0d1026..94432f4ddd40 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java @@ -25,6 +25,7 @@ public class AdvancedTestDataComparator implements TestDataComparator { public static final String AIRBYTE_DATE_FORMAT = "yyyy-MM-dd"; public static final String AIRBYTE_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; + public static final String AIRBYTE_DATETIME_PARSED_FORMAT = "yyyy-MM-dd HH:mm:ss.S"; public static final String AIRBYTE_DATETIME_WITH_TZ_FORMAT = "yyyy-MM-dd'T'HH:mm:ssXXX"; @Override diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java index 52cbba91356f..4cc27f9777e9 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java @@ -6,6 +6,8 @@ import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; @@ -30,7 +32,36 @@ protected boolean compareBooleanValues(String firstBooleanValue, String secondBo if (secondBooleanValue.equalsIgnoreCase("true") || secondBooleanValue.equalsIgnoreCase("false")) { return super.compareBooleanValues(firstBooleanValue, secondBooleanValue); } else { - return super.compareBooleanValues(firstBooleanValue, String.valueOf(secondBooleanValue.equals("1"))); + return super.compareBooleanValues(firstBooleanValue, + String.valueOf(secondBooleanValue.equals("1"))); + } + } + + @Override + protected boolean compareDateTimeValues(String expectedValue, String actualValue) { + var destinationDate = parseLocalDateTime(actualValue); + var expectedDate = LocalDate.parse(expectedValue, + DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT)); + return expectedDate.equals(destinationDate); + } + + + private LocalDate parseLocalDateTime(String dateTimeValue) { + if (dateTimeValue != null) { + return LocalDate.parse(dateTimeValue, + DateTimeFormatter.ofPattern(getFormat(dateTimeValue))); + } else { + return null; + } + } + + private String getFormat(String dateTimeValue) { + if (dateTimeValue.contains("T")) { + // MySql stores array of objects as a jsonb type, i.e. array of string for all cases + return AIRBYTE_DATETIME_FORMAT; + } else { + // MySql stores datetime as datetime type after normalization + return AIRBYTE_DATETIME_PARSED_FORMAT; } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java index a81ad0003920..347b704ee249 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java @@ -18,7 +18,6 @@ public class PostgresTestDataComparator extends AdvancedTestDataComparator { private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); - private static final String AIRBYTE_DATETIME_PARSED_FORMAT = "yyyy-MM-dd HH:mm:ss.S"; private static final String POSTGRES_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; private static final String POSTGRES_DATETIME_WITH_TZ_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; From c916c5b496dbfe9be388cfd6ca0bb76daff7c979 Mon Sep 17 00:00:00 2001 From: Mark Berger Date: Fri, 5 Aug 2022 15:16:02 +0300 Subject: [PATCH 10/25] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20Move=20the?= =?UTF-8?q?=20cancel=20button=20outside=20the=20run=20click=20area=20(#149?= =?UTF-8?q?55)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Moved the cancel button outside the run click area --- .../src/components/JobItem/JobItem.tsx | 5 +- .../components/AttemptDetails.module.scss | 7 + .../JobItem/components/AttemptDetails.tsx | 24 +- .../JobItem/components/LogsDetails.tsx | 3 +- .../JobItem/components/MainInfo.module.scss | 83 +++++++ .../JobItem/components/MainInfo.tsx | 210 ++++++------------ .../src/components/icons/RotateIcon.tsx | 14 ++ airbyte-webapp/src/locales/en.json | 4 +- .../components/StatusView.module.scss | 49 ++++ .../components/StatusView.tsx | 178 ++++++++------- 10 files changed, 333 insertions(+), 244 deletions(-) create mode 100644 airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss create mode 100644 airbyte-webapp/src/components/JobItem/components/MainInfo.module.scss create mode 100644 airbyte-webapp/src/components/icons/RotateIcon.tsx create mode 100644 airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.module.scss diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 27ae560a247f..df7d6006ebd3 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -55,12 +55,13 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { const { jobId: linkedJobId } = useAttemptLink(); const [isOpen, setIsOpen] = useState(linkedJobId === getJobId(job)); const scrollAnchor = useRef(null); + + const didSucceed = didJobSucceed(job); + const onExpand = () => { setIsOpen(!isOpen); }; - const didSucceed = didJobSucceed(job); - useEffectOnce(() => { if (linkedJobId) { scrollAnchor.current?.scrollIntoView({ diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss new file mode 100644 index 000000000000..55529ef48198 --- /dev/null +++ b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss @@ -0,0 +1,7 @@ +@use "../../../scss/colors"; + +.details { + font-size: 12px; + line-height: 15px; + color: colors.$grey; +} diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx index 5cc88e6eee10..d61106a11c97 100644 --- a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx @@ -1,11 +1,12 @@ +import classNames from "classnames"; import dayjs from "dayjs"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import styled from "styled-components"; import Status from "core/statuses"; import { AttemptRead, JobConfigType } from "../../../core/request/AirbyteClient"; +import styles from "./AttemptDetails.module.scss"; interface IProps { className?: string; @@ -13,15 +14,6 @@ interface IProps { configType?: JobConfigType; } -const Details = styled.div` - font-size: 12px; - line-height: 15px; - color: ${({ theme }) => theme.greyColor40}; -`; -const FailureReasonDetails = styled.div` - padding-bottom: 10px; -`; - const getFailureFromAttempt = (attempt: AttemptRead) => { return attempt.failureSummary && attempt.failureSummary.failures[0]; }; @@ -31,9 +23,9 @@ const AttemptDetails: React.FC = ({ attempt, className, configType }) => if (attempt.status !== Status.SUCCEEDED && attempt.status !== Status.FAILED) { return ( -
+
-
+ ); } @@ -77,7 +69,7 @@ const AttemptDetails: React.FC = ({ attempt, className, configType }) => const isFailed = attempt.status === Status.FAILED; return ( -
+
{formatBytes(attempt?.bytesSynced)} | @@ -107,7 +99,7 @@ const AttemptDetails: React.FC = ({ attempt, className, configType }) => ) : null}
{isFailed && ( - +
{formatMessage( { id: "ui.keyValuePairV3", @@ -117,9 +109,9 @@ const AttemptDetails: React.FC = ({ attempt, className, configType }) => value: getExternalFailureMessage(attempt), } )} - +
)} -
+ ); }; diff --git a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx b/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx index 07bd5682c928..1d29a5df5471 100644 --- a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx @@ -18,8 +18,7 @@ const LogHeader = styled.div` `; const AttemptDetailsSection = styled.div` - padding-left: 10px; - padding-top: 10px; + padding: 10px 0 10px 10px; `; const LogPath = styled.span` diff --git a/airbyte-webapp/src/components/JobItem/components/MainInfo.module.scss b/airbyte-webapp/src/components/JobItem/components/MainInfo.module.scss new file mode 100644 index 000000000000..c2fb8342106d --- /dev/null +++ b/airbyte-webapp/src/components/JobItem/components/MainInfo.module.scss @@ -0,0 +1,83 @@ +@use "../../../scss/colors"; +@use "../../../scss/variables"; + +.mainView { + cursor: pointer; + min-height: 75px; + + .titleCell { + display: flex; + color: colors.$dark-blue; + + .statusIcon { + display: flex; + align-items: center; + justify-content: center; + width: 75px; + + & > div { + margin: 0; + } + } + + .lastAttempt { + font-size: 12px; + font-weight: bold; + color: colors.$grey; + } + + .justification { + display: flex; + flex-direction: column; + justify-content: center; + } + } + + .timestampCell { + display: flex; + justify-content: flex-end; + align-items: center; + + .attemptCount { + font-size: 12px; + line-height: 15px; + color: colors.$red; + } + + .arrow { + transform: rotate(-90deg); + transition: variables.$transition; + opacity: 0; + color: colors.$dark-blue-50; + font-size: 22px; + margin: 0 30px 0 50px; + + div:hover > div > &, + div:hover > div > div > &, + div:hover > & { + opacity: 1; + } + } + } + + &.open { + .arrow { + transform: rotate(-0deg); + } + } + + &.failed { + .arrow, + .lastAttempt { + color: colors.$red; + } + } + + &.open:not(.failed) { + border-bottom: variables.$border-thin solid colors.$grey-100; + } + + &.open.failed { + border-bottom: variables.$border-thin solid colors.$red-50; + } +} diff --git a/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx b/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx index eb2146832919..117a471ab88a 100644 --- a/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx +++ b/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx @@ -1,85 +1,19 @@ import { faAngleDown } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import classNames from "classnames"; import React from "react"; import { FormattedDateParts, FormattedMessage, FormattedTimeParts } from "react-intl"; -import styled from "styled-components"; -import { LoadingButton, StatusIcon } from "components"; +import { StatusIcon } from "components"; import { Cell, Row } from "components/SimpleTableComponents"; import { AttemptRead, JobStatus } from "core/request/AirbyteClient"; import { SynchronousJobReadWithStatus } from "core/request/LogsRequestError"; -import useLoadingState from "hooks/useLoadingState"; import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/components/JobsList"; -import { useCancelJob } from "services/job/JobService"; -import { getJobId, getJobStatus } from "../JobItem"; +import { getJobStatus } from "../JobItem"; import AttemptDetails from "./AttemptDetails"; - -const MainView = styled(Row)<{ - isOpen?: boolean; - isFailed?: boolean; -}>` - cursor: pointer; - height: 75px; - padding: 15px 44px 10px 40px; - justify-content: space-between; - border-bottom: 1px solid - ${({ theme, isOpen, isFailed }) => (!isOpen ? "none" : isFailed ? theme.dangerTransparentColor : theme.greyColor20)}; -`; - -const Title = styled.div<{ isFailed?: boolean }>` - position: relative; - color: ${({ theme, isFailed }) => (isFailed ? theme.dangerColor : theme.darkPrimaryColor)}; -`; - -const ErrorSign = styled(StatusIcon)` - position: absolute; - left: -30px; -`; - -const AttemptCount = styled.div` - font-size: 12px; - line-height: 15px; - color: ${({ theme }) => theme.dangerColor}; -`; - -const CancelButton = styled(LoadingButton)` - margin-right: 10px; - padding: 3px 7px; - z-index: 1; -`; - -const InfoCell = styled(Cell)` - flex: none; -`; - -const Arrow = styled.div<{ - isOpen?: boolean; - isFailed?: boolean; -}>` - transform: ${({ isOpen }) => !isOpen && "rotate(-90deg)"}; - transition: 0.3s; - font-size: 22px; - line-height: 22px; - height: 22px; - color: ${({ theme, isFailed }) => (isFailed ? theme.dangerColor : theme.darkPrimaryColor)}; - position: absolute; - right: 18px; - top: calc(50% - 11px); - opacity: 0; - - div:hover > div > &, - div:hover > div > div > &, - div:hover > & { - opacity: 1; - } -`; -const Text = styled.div` - font-size: 12px; - font-weight: bold; - color: ${({ theme }) => theme.greyColor40}; -`; +import styles from "./MainInfo.module.scss"; const getJobConfig = (job: SynchronousJobReadWithStatus | JobsWithJobs) => (job as SynchronousJobReadWithStatus).configType ?? (job as JobsWithJobs).job.configType; @@ -87,100 +21,86 @@ const getJobConfig = (job: SynchronousJobReadWithStatus | JobsWithJobs) => const getJobCreatedAt = (job: SynchronousJobReadWithStatus | JobsWithJobs) => (job as SynchronousJobReadWithStatus).createdAt ?? (job as JobsWithJobs).job.createdAt; +const partialSuccessCheck = (attempts: AttemptRead[]) => { + if (attempts.length > 0 && attempts[attempts.length - 1].status === JobStatus.failed) { + return attempts.some((attempt) => attempt.failureSummary && attempt.failureSummary.partialSuccess); + } + return false; +}; + interface MainInfoProps { job: SynchronousJobReadWithStatus | JobsWithJobs; attempts?: AttemptRead[]; isOpen?: boolean; onExpand: () => void; isFailed?: boolean; - isPartialSuccess?: boolean; shortInfo?: boolean; } -const MainInfo: React.FC = ({ - job, - attempts = [], - isOpen, - onExpand, - isFailed, - shortInfo, - isPartialSuccess, -}) => { - const { isLoading, showFeedback, startAction } = useLoadingState(); - const cancelJob = useCancelJob(); - - const onCancelJob = (event: React.SyntheticEvent) => { - event.stopPropagation(); - const jobId = Number(getJobId(job)); - return startAction({ action: () => cancelJob(jobId) }); - }; - +const MainInfo: React.FC = ({ job, attempts = [], isOpen, onExpand, isFailed, shortInfo }) => { const jobStatus = getJobStatus(job); - const isNotCompleted = - jobStatus === JobStatus.pending || jobStatus === JobStatus.running || jobStatus === JobStatus.incomplete; - - const jobStatusLabel = isPartialSuccess ? ( - - ) : ( - - ); - - const getIcon = () => { - if (isPartialSuccess) { - return ; - } else if (isFailed && !shortInfo) { - return ; + const isPartialSuccess = partialSuccessCheck(attempts); + + const statusIcon = () => { + switch (true) { + case jobStatus === JobStatus.cancelled: + return ; + case jobStatus === JobStatus.running: + return ; + case jobStatus === JobStatus.succeeded: + return ; + case isPartialSuccess: + return ; + case !isPartialSuccess && isFailed && !shortInfo: + return ; + default: + return null; } - return null; }; return ( - - - - {getIcon()} - {jobStatusLabel} - {shortInfo ? <FormattedMessage id="sources.additionLogs" /> : null} - {attempts.length && !shortInfo ? ( - <div> + <Row + className={classNames(styles.mainView, { [styles.failed]: isFailed, [styles.open]: isOpen })} + onClick={onExpand} + > + <Cell className={styles.titleCell}> + <div className={styles.statusIcon}>{statusIcon()}</div> + <div className={styles.justification}> + {isPartialSuccess ? ( + <FormattedMessage id="sources.partialSuccess" /> + ) : ( + <FormattedMessage id={`sources.${getJobStatus(job)}`} /> + )} + {shortInfo && <FormattedMessage id="sources.additionLogs" />} + {attempts.length && !shortInfo && ( + <> {attempts.length > 1 && ( - <Text> + <div className={styles.lastAttempt}> <FormattedMessage id="sources.lastAttempt" /> - </Text> + </div> )} <AttemptDetails attempt={attempts[attempts.length - 1]} configType={getJobConfig(job)} /> + </> + )} + </div> + </Cell> + <Cell className={styles.timestampCell}> + <div> + <FormattedTimeParts value={getJobCreatedAt(job) * 1000} hour="numeric" minute="2-digit"> + {(parts) => <span>{`${parts[0].value}:${parts[2].value}${parts[4].value} `}</span>} + </FormattedTimeParts> + <FormattedDateParts value={getJobCreatedAt(job) * 1000} month="2-digit" day="2-digit"> + {(parts) => <span>{`${parts[0].value}/${parts[2].value}`}</span>} + </FormattedDateParts> + {attempts.length > 1 && ( + <div className={styles.attemptCount}> + <FormattedMessage id="sources.countAttempts" values={{ count: attempts.length }} /> </div> - ) : null} - - - - {!shortInfo && isNotCompleted && ( - - - - )} - - {(parts) => {`${parts[0].value}:${parts[2].value}${parts[4].value} `}} - - - {(parts) => {`${parts[0].value}/${parts[2].value}`}} - - {attempts.length > 1 && ( - - - - )} - - - - - + )} + + + + ); }; diff --git a/airbyte-webapp/src/components/icons/RotateIcon.tsx b/airbyte-webapp/src/components/icons/RotateIcon.tsx new file mode 100644 index 000000000000..00af9a51eb62 --- /dev/null +++ b/airbyte-webapp/src/components/icons/RotateIcon.tsx @@ -0,0 +1,14 @@ +import { theme } from "theme"; + +interface Props { + color?: string; +} + +export const RotateIcon = ({ color = theme.greyColor20 }: Props) => ( + + + +); diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index a544ac98307c..fe617f37d353 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -250,7 +250,6 @@ "sources.settings": "Settings", "sources.syncHistory": "Sync History", "sources.syncNow": "Sync now", - "sources.syncingNow": "Syncing now!", "sources.source": "Source", "sources.noSync": "No sync yet", "sources.emptySchema": "Schema is empty", @@ -356,6 +355,9 @@ "connection.linkCopied": "Link copied!", "connection.copyLogLink": "Copy link to log", "connection.connectionDeletedView": "This connection has been deleted. You can’t make any changes or run syncs.", + "connection.cancelSync": "Cancel Sync", + "connection.cancelReset": "Cancel Reset", + "connection.canceling": "Canceling...", "form.frequency": "Replication frequency*", "form.frequency.placeholder": "Select a frequency", diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.module.scss new file mode 100644 index 000000000000..0bd3b78ca9ac --- /dev/null +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.module.scss @@ -0,0 +1,49 @@ +@use "../../../../../scss/colors"; + +.statusView { + margin: 0 10px; + + .title { + display: flex; + justify-content: space-between; + flex-direction: row; + align-items: center; + + .actions { + display: flex; + + .resetButton, + .cancelButton, + .syncButton { + display: flex; + align-items: center; + justify-content: center; + } + + .resetButton { + margin-right: 10px; + } + + .cancelButton { + border-color: transparent; + background-color: colors.$red; + + .iconXmark { + margin-right: 12px; + font-size: 18px; + } + } + + .syncButton { + .iconRotate { + display: flex; + margin-right: 12px; + } + } + } + } + + .contentCard { + margin-bottom: 20px; + } +} diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx index 7cc3f06a4541..e465a223c3a2 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx @@ -1,64 +1,72 @@ -import { faRedoAlt } from "@fortawesome/free-solid-svg-icons"; +import { faXmark } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React from "react"; +import React, { useEffect, useState } from "react"; import { FormattedMessage } from "react-intl"; -import styled from "styled-components"; -import { Button, ContentCard, LoadingButton } from "components"; +import { Button, ContentCard } from "components"; import EmptyResource from "components/EmptyResourceBlock"; +import { RotateIcon } from "components/icons/RotateIcon"; import ToolTip from "components/ToolTip"; -import { ConnectionStatus, WebBackendConnectionRead } from "core/request/AirbyteClient"; +import { ConnectionStatus, JobWithAttemptsRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; import Status from "core/statuses"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useResetConnection, useSyncConnection } from "hooks/services/useConnectionHook"; -import useLoadingState from "hooks/useLoadingState"; -import { useListJobs } from "services/job/JobService"; +import { useCancelJob, useListJobs } from "services/job/JobService"; import JobsList from "./JobsList"; +import styles from "./StatusView.module.scss"; + +enum ActionType { + RESET = "reset_connection", + SYNC = "sync", +} + +interface ActiveJob { + id: number; + action: ActionType; + isCanceling: boolean; +} interface StatusViewProps { connection: WebBackendConnectionRead; isStatusUpdating?: boolean; } -const StyledContentCard = styled(ContentCard)` - margin-bottom: 20px; -`; - -const Title = styled.div` - display: flex; - justify-content: space-between; - flex-direction: row; - align-items: center; -`; - -const TryArrow = styled(FontAwesomeIcon)` - margin: 0 10px -1px 0; - font-size: 14px; -`; - -const SyncButton = styled(LoadingButton)` - padding: 5px 8px; - margin: -5px 0 -5px 9px; - min-width: 101px; - min-height: 28px; -`; - -const StatusView: React.FC = ({ connection, isStatusUpdating }) => { - const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); - const { isLoading, showFeedback, startAction } = useLoadingState(); - const allowSync = useFeature(FeatureItem.AllowSync); +const getJobRunningOrPending = (jobs: JobWithAttemptsRead[]) => { + return jobs.find((jobWithAttempts) => { + const jobStatus = jobWithAttempts?.job?.status; + return jobStatus === Status.PENDING || jobStatus === Status.RUNNING || jobStatus === Status.INCOMPLETE; + }); +}; + +const StatusView: React.FC = ({ connection }) => { + const [activeJob, setActiveJob] = useState(); const jobs = useListJobs({ configId: connection.connectionId, configTypes: ["sync", "reset_connection"], }); - const isAtLeastOneJobRunningOrPending = jobs.some((jobWithAttempts) => { - const status = jobWithAttempts?.job?.status; - return status === Status.RUNNING || status === Status.PENDING; - }); + + useEffect(() => { + const jobRunningOrPending = getJobRunningOrPending(jobs); + + setActiveJob( + (state) => + ({ + id: jobRunningOrPending?.job?.id, + action: jobRunningOrPending?.job?.configType, + isCanceling: state?.isCanceling && !!jobRunningOrPending, + // We need to disable button when job is canceled but the job list still has a running job + } as ActiveJob) + ); + }, [jobs]); + + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + + const allowSync = useFeature(FeatureItem.AllowSync); + const cancelJob = useCancelJob(); const { mutateAsync: resetConnection } = useResetConnection(); const { mutateAsync: syncConnection } = useSyncConnection(); @@ -74,56 +82,70 @@ const StatusView: React.FC = ({ connection, isStatusUpdating }) cancelButtonText: "form.noNeed", onSubmit: async () => { await onReset(); + setActiveJob((state) => ({ ...state, action: ActionType.RESET } as ActiveJob)); closeConfirmationModal(); }, submitButtonDataId: "reset", }); }; - const resetDataBtn = ( - - ); + const onSyncNowButtonClick = () => { + setActiveJob((state) => ({ ...state, action: ActionType.SYNC } as ActiveJob)); + return onSync(); + }; - const syncNowBtn = ( - startAction({ action: onSync })} - > - {showFeedback ? ( - - ) : ( - <> - - - - )} - + const onCancelJob = () => { + if (!activeJob?.id) { + return; + } + setActiveJob((state) => ({ ...state, isCanceling: true } as ActiveJob)); + return cancelJob(activeJob.id); + }; + + const cancelJobBtn = ( + ); return ( - - - {connection.status === ConnectionStatus.active && ( -
- - - - - - -
- )} - - } - > - {jobs.length ? : } />} - +
+ + + {connection.status === ConnectionStatus.active && ( +
+ {!activeJob?.action && ( + <> + + + + )} + {activeJob?.action && !activeJob.isCanceling && cancelJobBtn} + {activeJob?.action && activeJob.isCanceling && ( + + + + )} +
+ )} +
+ } + > + {jobs.length ? : } />} + + ); }; From 823a79b69a4a409b57b86d146f4a731223088b25 Mon Sep 17 00:00:00 2001 From: oneshcheret <33333155+sashaNeshcheret@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:21:56 +0300 Subject: [PATCH 11/25] S3, Databricks and Gcs destinations fix test and publish (#15360) * Postgres source added items for array data type * Postgres source updated tests for array data type * S3 destination fix key pair oauth test * S3 destination clean code * S3 destination bump version * S3 destination bump version * Databricks and gcs destinations bump versions * auto-bump connector version [ci skip] * auto-bump connector version [ci skip] * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../src/main/resources/seed/destination_definitions.yaml | 6 +++--- .../init/src/main/resources/seed/destination_specs.yaml | 6 +++--- .../main/resources/number_data_type_array_test_messages.txt | 2 +- .../connectors/destination-databricks/Dockerfile | 2 +- airbyte-integrations/connectors/destination-gcs/Dockerfile | 2 +- airbyte-integrations/connectors/destination-s3/Dockerfile | 2 +- docs/integrations/destinations/databricks.md | 1 + docs/integrations/destinations/gcs.md | 2 +- docs/integrations/destinations/s3.md | 2 +- 9 files changed, 13 insertions(+), 12 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index ca13c26351f0..3d754ff0cfcd 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -73,7 +73,7 @@ - name: Databricks Lakehouse destinationDefinitionId: 072d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-databricks - dockerImageTag: 0.2.5 + dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/destinations/databricks icon: databricks.svg releaseStage: alpha @@ -106,7 +106,7 @@ - name: Google Cloud Storage (GCS) destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a dockerRepository: airbyte/destination-gcs - dockerImageTag: 0.2.9 + dockerImageTag: 0.2.10 documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs icon: googlecloudstorage.svg resourceRequirements: @@ -250,7 +250,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.3.11 + dockerImageTag: 0.3.12 documentationUrl: https://docs.airbyte.io/integrations/destinations/s3 icon: s3.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index fb89aed4be8a..ddd9df65fe72 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -966,7 +966,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-databricks:0.2.5" +- dockerImage: "airbyte/destination-databricks:0.2.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/databricks" connectionSpecification: @@ -1563,7 +1563,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-gcs:0.2.9" +- dockerImage: "airbyte/destination-gcs:0.2.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs" connectionSpecification: @@ -3974,7 +3974,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.3.11" +- dockerImage: "airbyte/destination-s3:0.3.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3" connectionSpecification: diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_messages.txt index 11662ae71ffc..13714bd4e702 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_messages.txt +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_messages.txt @@ -1,2 +1,2 @@ -{"type": "RECORD", "record": {"stream": "array_test_1", "emitted_at": 1602637589100, "data": { "array_number" : [-12345.678, 100000000000000000.1234],"array_float" : [-12345.678, 0, 1000000000000000000000000000000000000000000000000000.1234], "array_integer" : [42, 0, 12345]}} +{"type": "RECORD", "record": {"stream": "array_test_1", "emitted_at": 1602637589100, "data": { "array_number" : [-12345.678, 100000000000000000.1234],"array_float" : [-12345.678, 0, 1000000000000000000000000000000000000000000000000000.1234], "array_integer" : [42, 0, 12345]}}} {"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/connectors/destination-databricks/Dockerfile b/airbyte-integrations/connectors/destination-databricks/Dockerfile index 92521ca3420b..d12822925308 100644 --- a/airbyte-integrations/connectors/destination-databricks/Dockerfile +++ b/airbyte-integrations/connectors/destination-databricks/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-databricks COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/destination-databricks diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile index 1d2b3725e12a..8d8c06e21060 100644 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ b/airbyte-integrations/connectors/destination-gcs/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 9efc767675cd..d00a1aa7085e 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-s3 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.11 +LABEL io.airbyte.version=0.3.12 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/docs/integrations/destinations/databricks.md b/docs/integrations/destinations/databricks.md index ee6e9de3b1b0..9853cd8b6e86 100644 --- a/docs/integrations/destinations/databricks.md +++ b/docs/integrations/destinations/databricks.md @@ -105,6 +105,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | |:--------|:-----------|:--------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------| +| 0.2.6 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | | 0.2.5 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | | 0.2.4 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors | | 0.2.3 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index a72542e766ec..205b587ead22 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -235,7 +235,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | -| (unpublished) | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | +| 0.2.10 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | | 0.2.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | | 0.2.8 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | | 0.2.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index aa6e1ee4a0a4..af5a4798793b 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -320,7 +320,7 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| -| (unpublished) | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | +| 0.3.12 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | | 0.3.11 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | | 0.3.10 | 2022-06-30 | [\#14332](https://github.com/airbytehq/airbyte/pull/14332) | Change INSTANCE_PROFILE to use `AWSDefaultProfileCredential`, which supports more authentications on AWS | | 0.3.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | From 95aeb39325f8f91526dd4f01401fc2c00d9af1cf Mon Sep 17 00:00:00 2001 From: Greg Solovyev Date: Fri, 5 Aug 2022 09:13:24 -0700 Subject: [PATCH 12/25] Fix typo in change log (#15343) * Fix typo in several change logs Co-authored-by: andrii.leonets --- docs/integrations/destinations/bigquery.md | 30 +++++++++++----------- docs/integrations/destinations/gcs.md | 4 +-- docs/integrations/destinations/s3.md | 2 +- docs/integrations/sources/mssql.md | 16 ++++++------ docs/integrations/sources/mysql.md | 20 +++++++-------- docs/integrations/sources/postgres.md | 2 +- 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 01ba5b695fd0..8f1aca18d2ea 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -133,15 +133,15 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------| -| 1.1.14 | 2022-08-03 | [14784](https://github.com/airbytehq/airbyte/pull/14784) | Enabling Application Default Credentials | -| 1.1.13 | 2022-08-02 | [15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode | -| 1.1.12 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | -| 1.1.11 | 2022-06-24 | [14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | -| 1.1.10 | 2022-06-16 | [13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | -| 1.1.9 | 2022-06-17 | [13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | -| 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | +| 1.1.14 | 2022-08-03 | [14784](https://github.com/airbytehq/airbyte/pull/14784) | Enabling Application Default Credentials | +| 1.1.13 | 2022-08-02 | [15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode | +| 1.1.12 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 1.1.11 | 2022-06-24 | [14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | +| 1.1.10 | 2022-06-16 | [13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | +| 1.1.9 | 2022-06-17 | [13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | +| 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | | 1.1.7 | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | -| 1.1.6 | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | +| 1.1.6 | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | | 1.1.5 | 2022-05-12 | [12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessage on error. | | 1.1.4 | 2022-05-04 | [12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | | 1.1.3 | 2022-05-02 | [12528](https://github.com/airbytehq/airbyte/pull/12528) | Update Dataset location field description | @@ -177,13 +177,13 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| -| 1.1.14 | 2022-08-02 | [15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode | -| 1.1.13 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | -| 1.1.12 | 2022-06-29 | [14079](https://github.com/airbytehq/airbyte/pull/14079) | Map "airbyte_type": "big_integer" to INT64 | -| 1.1.11 | 2022-06-24 | [14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | -| 1.1.10 | 2022-06-16 | [13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | -| 1.1.9 | 2022-06-17 | [13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | -| 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | +| 1.1.14 | 2022-08-02 | [15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode | +| 1.1.13 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 1.1.12 | 2022-06-29 | [14079](https://github.com/airbytehq/airbyte/pull/14079) | Map "airbyte_type": "big_integer" to INT64 | +| 1.1.11 | 2022-06-24 | [14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | +| 1.1.10 | 2022-06-16 | [13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | +| 1.1.9 | 2022-06-17 | [13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | +| 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | | 1.1.7 | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | | 1.1.6 | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | | 0.3.5 | 2022-05-12 | [12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessage on error. | diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index 205b587ead22..c2a82624bbe7 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -235,7 +235,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | -| 0.2.10 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | +| 0.2.10 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.2.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | | 0.2.8 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | | 0.2.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | @@ -258,4 +258,4 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | 0.1.13 | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288) | Support Json `additionalProperties`. | | 0.1.2 | 2021-09-12 | [\#5720](https://github.com/airbytehq/airbyte/issues/5720) | Added configurable block size for stream. Each stream is limited to 10,000 by GCS | | 0.1.1 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added storing gcsCsvFileLocation property for CSV format. This is used by destination-bigquery \(GCS Staging upload type\) | -| 0.1.0 | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784) | Initial release. | +| 0.1.0 | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784) | Initial release. | diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index af5a4798793b..b021c4427067 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -320,7 +320,7 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.3.12 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | +| 0.3.12 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.3.11 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | | 0.3.10 | 2022-06-30 | [\#14332](https://github.com/airbytehq/airbyte/pull/14332) | Change INSTANCE_PROFILE to use `AWSDefaultProfileCredential`, which supports more authentications on AWS | | 0.3.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index f86ef2d6f572..6c41a9509308 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -306,14 +306,14 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------------------------------------------------- |:-------------------------------------------------------------------------------------------------------| -| 0.4.13 | 2022-08-04 | [15268](https://github.com/airbytehq/airbyte/pull/15268) | Added [] enclosing to escape special character in the database name| -| 0.4.12 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | -| 0.4.11 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.4.10 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.4.9 | 2022-07-05 | [14379](https://github.com/airbytehq/airbyte/pull/14379) | Aligned Normal and CDC migration + added some fixes for datatypes processing | -| 0.4.8 | 2022-06-24 | [14121](https://github.com/airbytehq/airbyte/pull/14121) | Omit using 'USE' keyword on Azure SQL with CDC | -| 0.4.5 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | -| 0.4.3 | 2022-06-17 | [13887](https://github.com/airbytehq/airbyte/pull/13887) | Increase version to include changes from [13854](https://github.com/airbytehq/airbyte/pull/13854) | +| 0.4.13 | 2022-08-04 | [15268](https://github.com/airbytehq/airbyte/pull/15268) | Added [] enclosing to escape special character in the database name | +| 0.4.12 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.4.11 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.4.10 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.4.9 | 2022-07-05 | [14379](https://github.com/airbytehq/airbyte/pull/14379) | Aligned Normal and CDC migration + added some fixes for datatypes processing | +| 0.4.8 | 2022-06-24 | [14121](https://github.com/airbytehq/airbyte/pull/14121) | Omit using 'USE' keyword on Azure SQL with CDC | +| 0.4.5 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | +| 0.4.3 | 2022-06-17 | [13887](https://github.com/airbytehq/airbyte/pull/13887) | Increase version to include changes from [13854](https://github.com/airbytehq/airbyte/pull/13854) | | 0.4.2 | 2022-06-06 | [13435](https://github.com/airbytehq/airbyte/pull/13435) | Adjust JDBC fetch size based on max memory and max row size | | 0.4.1 | 2022-05-25 | [13419](https://github.com/airbytehq/airbyte/pull/13419) | Correct enum for Standard method. | | 0.4.0 | 2022-05-25 | [12759](https://github.com/airbytehq/airbyte/pull/12759) [13168](https://github.com/airbytehq/airbyte/pull/13168) | For CDC, Add option to ignore existing data and only sync new changes from the database. | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index e0b2f705c0d4..537e0338005d 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -185,16 +185,16 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| -| 0.6.1 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | -| 0.6.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | -| 0.5.17 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.5.16 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.5.15 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | -| 0.5.13 | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945) | Aligned datatype test | -| 0.5.12 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.5.11 | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544) | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads. | -| 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.6.1 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.6.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | +| 0.5.17 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.5.16 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.5.15 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | +| 0.5.13 | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945) | Aligned datatype test | +| 0.5.12 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.5.11 | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544) | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads. | +| 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.5.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.5.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | | 0.5.4 | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251) | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 9b397e701bd5..d2ca2e81ec91 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -359,7 +359,7 @@ Possible solutions include: | | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently | | 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | | | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | -| 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiply log bindings | +| 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | | 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | | 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (â›” this version has a bug and will not work; use `0.4.42` instead) | From 2929848d6ddaa5e109ea0631678b1b109cbdbb0f Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Fri, 5 Aug 2022 18:23:49 +0200 Subject: [PATCH 13/25] Fix connection settings changing randomly (#15332) * Fix connection settings changing randomly * Update airbyte-webapp/src/locales/en.json Co-authored-by: Andy Jih Co-authored-by: Andy Jih --- .../CreateConnectionContent.tsx | 13 +--- airbyte-webapp/src/locales/en.json | 3 + .../components/ReplicationView.tsx | 64 +++++++++++-------- .../ConnectionForm/ConnectionForm.tsx | 33 +++++----- .../Connection/ConnectionForm/formConfig.tsx | 2 +- 5 files changed, 60 insertions(+), 55 deletions(-) diff --git a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx b/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx index 83221396a6f1..6c2108711a77 100644 --- a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx +++ b/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx @@ -1,4 +1,4 @@ -import React, { Suspense, useMemo, useState } from "react"; +import React, { Suspense, useMemo } from "react"; import styled from "styled-components"; import { ContentCard } from "components"; @@ -12,7 +12,6 @@ import { useAnalyticsService } from "hooks/services/Analytics"; import { useCreateConnection, ValuesProps } from "hooks/services/useConnectionHook"; import ConnectionForm from "views/Connection/ConnectionForm"; import { ConnectionFormProps } from "views/Connection/ConnectionForm/ConnectionForm"; -import { FormikConnectionFormValues } from "views/Connection/ConnectionForm/formConfig"; import { DestinationRead, SourceRead, WebBackendConnectionRead } from "../../core/request/AirbyteClient"; import { useDiscoverSchema } from "../../hooks/services/useSourceHook"; @@ -45,21 +44,14 @@ const CreateConnectionContent: React.FC = ({ const { schema, isLoading, schemaErrorStatus, catalogId, onDiscoverSchema } = useDiscoverSchema(source.sourceId); - const [connectionFormValues, setConnectionFormValues] = useState(); - const connection = useMemo( () => ({ - name: connectionFormValues?.name ?? "", - namespaceDefinition: connectionFormValues?.namespaceDefinition, - namespaceFormat: connectionFormValues?.namespaceFormat, - prefix: connectionFormValues?.prefix, - schedule: connectionFormValues?.schedule ?? undefined, syncCatalog: schema, destination, source, catalogId, }), - [connectionFormValues, schema, destination, source, catalogId] + [schema, destination, source, catalogId] ); const onSubmitConnectionStep = async (values: ValuesProps) => { @@ -124,7 +116,6 @@ const CreateConnectionContent: React.FC = ({ additionBottomControls={additionBottomControls} onDropDownSelect={onSelectFrequency} onSubmit={onSubmitConnectionStep} - onChangeValues={setConnectionFormValues} /> ); diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index fe617f37d353..8e1e5ca7dbe9 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -329,6 +329,9 @@ "connection.sourceTestAgain": "Test source connection again", "connection.resetData": "Reset your data", "connection.updateSchema": "Refresh source schema", + "connection.updateSchema.formChanged.title": "Unsaved changes", + "connection.updateSchema.formChanged.text": "Your replication settings have unsaved changes. Those will be lost when refreshing the source schema. Save your changes before refreshing the source schema to not lose them.", + "connection.updateSchema.formChanged.confirm": "Discard changes and refresh schema", "connection.updateSchema.completed": "Refreshed source schema", "connection.updateSchema.confirm": "Confirm", "connection.updateSchema.new": "{value} new {item}", diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx index 0fd5827bb330..367433ef0ece 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx @@ -1,6 +1,6 @@ import { faSyncAlt } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React, { useMemo, useState } from "react"; +import React, { useCallback, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useAsyncFn, useUnmount } from "react-use"; import styled from "styled-components"; @@ -10,6 +10,7 @@ import LoadingSchema from "components/LoadingSchema"; import { toWebBackendConnectionUpdate } from "core/domain/connection"; import { ConnectionStateType, ConnectionStatus } from "core/request/AirbyteClient"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useModalService } from "hooks/services/Modal"; import { useConnectionLoad, @@ -21,7 +22,6 @@ import { equal } from "utils/objects"; import { CatalogDiffModal } from "views/Connection/CatalogDiffModal/CatalogDiffModal"; import ConnectionForm from "views/Connection/ConnectionForm"; import { ConnectionFormSubmitResult } from "views/Connection/ConnectionForm/ConnectionForm"; -import { FormikConnectionFormValues } from "views/Connection/ConnectionForm/formConfig"; interface ReplicationViewProps { onAfterSaveSchema: () => void; @@ -84,9 +84,10 @@ const TryArrow = styled(FontAwesomeIcon)` export const ReplicationView: React.FC = ({ onAfterSaveSchema, connectionId }) => { const { formatMessage } = useIntl(); const { openModal, closeModal } = useModalService(); + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + const connectionFormDirtyRef = useRef(false); const [activeUpdatingSchemaMode, setActiveUpdatingSchemaMode] = useState(false); const [saved, setSaved] = useState(false); - const [connectionFormValues, setConnectionFormValues] = useState(); const connectionService = useConnectionService(); const { mutateAsync: updateConnection } = useUpdateConnection(); @@ -97,28 +98,19 @@ export const ReplicationView: React.FC = ({ onAfterSaveSch [connectionId] ); - useUnmount(() => closeModal()); - - const connection = useMemo(() => { - if (activeUpdatingSchemaMode && connectionWithRefreshCatalog) { - // merge connectionFormValues (unsaved previous form state) with the refreshed connection data: - // 1. if there is a namespace definition, format, prefix, or schedule in connectionFormValues, - // use those and fill in the rest from the database - // 2. otherwise, use the values from the database - // 3. if none of the above, use the default values. - return { - ...connectionWithRefreshCatalog, - namespaceDefinition: - connectionFormValues?.namespaceDefinition ?? connectionWithRefreshCatalog.namespaceDefinition, - namespaceFormat: connectionFormValues?.namespaceFormat ?? connectionWithRefreshCatalog.namespaceFormat, - prefix: connectionFormValues?.prefix ?? connectionWithRefreshCatalog.prefix, - schedule: connectionFormValues?.schedule ?? connectionWithRefreshCatalog.schedule, - }; - } - return initialConnection; - }, [activeUpdatingSchemaMode, connectionWithRefreshCatalog, initialConnection, connectionFormValues]); + useUnmount(() => { + closeModal(); + closeConfirmationModal(); + }); + + const connection = activeUpdatingSchemaMode ? connectionWithRefreshCatalog : initialConnection; const saveConnection = async (values: ValuesProps, { skipReset }: { skipReset: boolean }) => { + if (!connection) { + // onSubmit should only be called while the catalog isn't currently refreshing at the moment, + // which is the only case when `connection` would be `undefined`. + return; + } const initialSyncSchema = connection.syncCatalog; const connectionAsUpdate = toWebBackendConnectionUpdate(connection); @@ -174,7 +166,7 @@ export const ReplicationView: React.FC = ({ onAfterSaveSch } }; - const onRefreshSourceSchema = async () => { + const refreshSourceSchema = async () => { setSaved(false); setActiveUpdatingSchemaMode(true); const { catalogDiff, syncCatalog } = await refreshCatalog(); @@ -189,11 +181,33 @@ export const ReplicationView: React.FC = ({ onAfterSaveSch } }; + const onRefreshSourceSchema = async () => { + if (connectionFormDirtyRef.current) { + // The form is dirty so we show a warning before proceeding. + openConfirmationModal({ + title: "connection.updateSchema.formChanged.title", + text: "connection.updateSchema.formChanged.text", + submitButtonText: "connection.updateSchema.formChanged.confirm", + onSubmit: () => { + closeConfirmationModal(); + refreshSourceSchema(); + }, + }); + } else { + // The form is not dirty so we can directly refresh the source schema. + refreshSourceSchema(); + } + }; + const onCancelConnectionFormEdit = () => { setSaved(false); setActiveUpdatingSchemaMode(false); }; + const onDirtyChanges = useCallback((dirty: boolean) => { + connectionFormDirtyRef.current = dirty; + }, []); + return ( {!isRefreshingCatalog && connection ? ( @@ -210,7 +224,7 @@ export const ReplicationView: React.FC = ({ onAfterSaveSch } - onChangeValues={setConnectionFormValues} + onFormDirtyChanges={onDirtyChanges} /> ) : ( diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx index d8ecdc15d004..9fa2eee3e5a5 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx @@ -1,8 +1,7 @@ -import { Field, FieldProps, Form, Formik, FormikHelpers, useFormikContext } from "formik"; -import React, { useCallback, useState } from "react"; +import { Field, FieldProps, Form, Formik, FormikHelpers } from "formik"; +import React, { useCallback, useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useToggle } from "react-use"; -import { useDebounce } from "react-use"; import styled from "styled-components"; import { Card, ControlLabels, DropDown, DropDownRow, H5, Input } from "components"; @@ -102,20 +101,18 @@ export interface ConnectionFormSubmitResult { export type ConnectionFormMode = "create" | "edit" | "readonly"; -// eslint-disable-next-line react/function-component-definition -function FormValuesChangeTracker({ onChangeValues }: { onChangeValues?: (values: T) => void }) { - // Grab values from context - const { values } = useFormikContext(); - useDebounce( - () => { - onChangeValues?.(values); - }, - 200, - [values, onChangeValues] - ); - return null; +interface DirtyChangeTrackerProps { + dirty: boolean; + onChanges: (dirty: boolean) => void; } +const DirtyChangeTracker: React.FC = ({ dirty, onChanges }) => { + useEffect(() => { + onChanges(dirty); + }, [dirty, onChanges]); + return null; +}; + interface ConnectionFormProps { onSubmit: (values: ConnectionFormValues) => Promise; className?: string; @@ -123,7 +120,7 @@ interface ConnectionFormProps { successMessage?: React.ReactNode; onDropDownSelect?: (item: DropDownRow.IDataItem) => void; onCancel?: () => void; - onChangeValues?: (values: FormikConnectionFormValues) => void; + onFormDirtyChanges?: (dirty: boolean) => void; /** Should be passed when connection is updated with withRefreshCatalog flag */ canSubmitUntouchedForm?: boolean; @@ -146,7 +143,7 @@ const ConnectionForm: React.FC = ({ canSubmitUntouchedForm, additionalSchemaControl, connection, - onChangeValues, + onFormDirtyChanges, }) => { const destDefinition = useGetDestinationDefinitionSpecification(connection.destination.destinationDefinitionId); const { clearFormChange } = useFormChangeTrackerService(); @@ -200,7 +197,7 @@ const ConnectionForm: React.FC = ({ {({ isSubmitting, setFieldValue, isValid, dirty, resetForm, values }) => ( - + {onFormDirtyChanges && } {!isEditMode && (
diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx index 2852f086004e..2e6f6e3e568f 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx @@ -234,7 +234,7 @@ const useInitialValues = ( const initialValues: FormikConnectionFormValues = { name: connection.name ?? `${connection.source.name} <> ${connection.destination.name}`, syncCatalog: initialSchema, - schedule: connection.connectionId || connection.schedule ? connection.schedule ?? null : DEFAULT_SCHEDULE, + schedule: connection.connectionId ? connection.schedule ?? null : DEFAULT_SCHEDULE, prefix: connection.prefix || "", namespaceDefinition: connection.namespaceDefinition || NamespaceDefinitionType.source, namespaceFormat: connection.namespaceFormat ?? SOURCE_NAMESPACE_TAG, From cf5d462610e3c2bd38997cc739a6f494f0c86ee3 Mon Sep 17 00:00:00 2001 From: Erica Struthers <93952107+erica-airbyte@users.noreply.github.com> Date: Fri, 5 Aug 2022 10:31:44 -0600 Subject: [PATCH 14/25] Update hubspot.md (#15369) --- docs/integrations/sources/hubspot.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 93e911b2d8a5..d773aa19fd64 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -4,7 +4,7 @@ This page guides you through setting up the HubSpot source connector. ## Prerequisite -You can use OAuth or an API key to authenticate your HubSpot account. If you choose to use OAuth, you need to configure the appropriate [scopes](https://legacydocs.hubspot.com/docs/methods/oauth2/initiate-oauth-integration#scopes) for the following streams: +You can use OAuth, API key, or Private App to authenticate your HubSpot account. If you choose to use OAuth or Private App, you need to configure the appropriate [scopes](https://legacydocs.hubspot.com/docs/methods/oauth2/initiate-oauth-integration#scopes) for the following streams: | Stream | Required Scope | |:----------------------------|:---------------------------------------------------------------------------------| From dfed48b71233f8bd29613d0fbe3e479a906b6306 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Fri, 5 Aug 2022 21:11:52 +0300 Subject: [PATCH 15/25] =?UTF-8?q?=F0=9F=AA=9F=20=20Add=20Segment=20call=20?= =?UTF-8?q?for=20Connection=20Delete=20(#15365)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/hooks/services/useConnectionHook.tsx | 17 ++++++++++---- .../ConnectionItemPage/ConnectionItemPage.tsx | 2 +- .../components/SettingsView.module.scss | 5 ++++ .../components/SettingsView.tsx | 23 ++++++++----------- 4 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.module.scss diff --git a/airbyte-webapp/src/hooks/services/useConnectionHook.tsx b/airbyte-webapp/src/hooks/services/useConnectionHook.tsx index 1d53ce8de3d7..9e2c640f624b 100644 --- a/airbyte-webapp/src/hooks/services/useConnectionHook.tsx +++ b/airbyte-webapp/src/hooks/services/useConnectionHook.tsx @@ -171,15 +171,24 @@ const useCreateConnection = () => { const useDeleteConnection = () => { const service = useConnectionService(); const queryClient = useQueryClient(); + const analyticsService = useAnalyticsService(); + + return useMutation((connection: WebBackendConnectionRead) => service.delete(connection.connectionId), { + onSuccess: (_data, connection) => { + analyticsService.track(Namespace.CONNECTION, Action.DELETE, { + actionDescription: "Connection deleted", + connector_source: connection.source?.sourceName, + connector_source_definition_id: connection.source?.sourceDefinitionId, + connector_destination: connection.destination?.destinationName, + connector_destination_definition_id: connection.destination?.destinationDefinitionId, + }); - return useMutation((connectionId: string) => service.delete(connectionId), { - onSuccess: (_data, connectionId) => { - queryClient.removeQueries(connectionsKeys.detail(connectionId)); + queryClient.removeQueries(connectionsKeys.detail(connection.connectionId)); queryClient.setQueryData( connectionsKeys.lists(), (lst: ListConnection | undefined) => ({ - connections: lst?.connections.filter((conn) => conn.connectionId !== connectionId) ?? [], + connections: lst?.connections.filter((conn) => conn.connectionId !== connection.connectionId) ?? [], } as ListConnection) ); }, diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx index 790f41f8cd7f..bfd9fb03dddb 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx @@ -87,7 +87,7 @@ const ConnectionItemPage: React.FC = () => { /> : } + element={isConnectionDeleted ? : } /> } /> diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.module.scss new file mode 100644 index 000000000000..432a7e8229a7 --- /dev/null +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.module.scss @@ -0,0 +1,5 @@ +.container { + max-width: 647px; + margin: 0 auto; + padding-bottom: 10px; +} diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx index f4afc29cca3a..d882015ccc88 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx @@ -1,32 +1,27 @@ import React from "react"; -import styled from "styled-components"; import DeleteBlock from "components/DeleteBlock"; import { useDeleteConnection } from "hooks/services/useConnectionHook"; +import { WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; +import styles from "./SettingsView.module.scss"; import { StateBlock } from "./StateBlock"; -interface IProps { - connectionId: string; +interface SettingsViewProps { + connection: WebBackendConnectionRead; } -const Content = styled.div` - max-width: 647px; - margin: 0 auto; - padding-bottom: 10px; -`; - -const SettingsView: React.FC = ({ connectionId }) => { +const SettingsView: React.FC = ({ connection }) => { const { mutateAsync: deleteConnection } = useDeleteConnection(); - const onDelete = () => deleteConnection(connectionId); + const onDelete = () => deleteConnection(connection); return ( - - +
+ - +
); }; From 1115637b917dd485e718701b9c57f517d6adb7f8 Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Fri, 5 Aug 2022 14:13:59 -0400 Subject: [PATCH 16/25] Fixed postgres formatting (#15370) --- docs/integrations/sources/postgres.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index d2ca2e81ec91..d1f4c1bab8df 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -353,6 +353,7 @@ Possible solutions include: ## Changelog | Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | | 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | | 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | | 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | From b9b064fa44c6ac96fd75c5fcf510b16bedd0d870 Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Fri, 5 Aug 2022 14:24:31 -0400 Subject: [PATCH 17/25] Updated postgres doc and fixed other minor nits (#15297) * Updated postgres doc and other minor nits * fixed formatting * added note about connectTimeout * added limits to the initial wait time values --- docs/integrations/README.md | 1 + docs/integrations/sources/glassfrog.md | 5 +---- docs/integrations/sources/postgres.md | 21 +++++++++++++++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index d626e9968d87..bef4347856fb 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -113,6 +113,7 @@ For more information about the grading system, see [Product Release Stages](http | [Oracle PeopleSoft](sources/oracle-peoplesoft.md) | Alpha | No | | [Oracle Siebel CRM](sources/oracle-siebel-crm.md) | Alpha | No | | [Orb](sources/orb.md) | Alpha | Yes | +| [Orbit](sources/orbit.md) | Alpha | No | | [Outreach](./sources/outreach.md) | Alpha | No | | [PagerDuty](sources/pagerduty.md) | Alpha | No | | [PayPal Transaction](sources/paypal-transaction.md) | Alpha | Yes | diff --git a/docs/integrations/sources/glassfrog.md b/docs/integrations/sources/glassfrog.md index dd5d1edf5555..08d3b2cc0ff5 100644 --- a/docs/integrations/sources/glassfrog.md +++ b/docs/integrations/sources/glassfrog.md @@ -1,4 +1,4 @@ -# Shortio +# Glassfrog ## Sync overview @@ -20,9 +20,6 @@ This Source is capable of syncing the following Streams: * [Roles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#d1f31f7a-1d42-8c86-be1d-a36e640bf993) - - - ### Data type mapping | Integration Type | Airbyte Type | Notes | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index d1f4c1bab8df..9d61d4b4a93a 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -102,13 +102,15 @@ This issue is tracked in [#9771](https://github.com/airbytehq/airbyte/issues/977 These parameters will be added at the end of the JDBC URL that the AirByte will use to connect to your Postgres database. + The connector now supports `connectTimeout` and defaults to 60 seconds. Setting connectTimeout to 0 seconds will set the timeout to the longest time available. + **Note:** Do not use the following keys in JDBC URL Params field as they will be overwritten by Airbyte: `currentSchema`, `user`, `password`, `ssl`, and `sslmode`. :::warning This is an advanced configuration option. Users are advised to use it with caution. ::: - + 9. For Airbyte OSS, toggle the switch to connect using SSL. Airbyte Cloud uses SSL by default. 10. For Replication Method, select Standard or [Logical CDC](https://www.postgresql.org/docs/10/logical-replication.html) from the dropdown. Refer to [Configuring Postgres connector with Change Data Capture (CDC)](#configuring-postgres-connector-with-change-data-capture-cdc) for more information. 11. For SSH Tunnel Method, select: @@ -238,7 +240,22 @@ Also, the publication should include all the tables and only the tables that nee The Airbyte UI currently allows selecting any tables for CDC. If a table is selected that is not part of the publication, it will not be replicated even though it is selected. If a table is part of the publication but does not have a replication identity, that replication identity will be created automatically on the first run if the Airbyte user has the necessary permissions. ::: -#### Step 5: Set up the Postgres source connector +#### Step 5: [Optional] Set up initial waiting time + +:::warning +This is an advanced feature. Use it if absolutely necessary. +::: + +The Postgres connector may need some time to start processing the data in the CDC mode in the following scenarios: + +- When the connection is set up for the first time and a snapshot is needed +- When the connector has a lot of change logs to process + +The connector waits for the default initial wait time of 5 minutes (300 seconds). Setting the parameter to a longer duration will result in slower syncs, while setting it to a shorter duration may cause the connector to not have enough time to create the initial snapshot or read through the change logs. The valid range is 120 seconds to 1200 seconds. + +If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs. + +#### Step 6: Set up the Postgres source connector In [Step 2](#step-2-set-up-the-postgres-connector-in-airbyte) of the connector setup guide, enter the replication slot and publication you just created. From 61ee04380ba9534555bd7c6663fe7fa6e85d6d69 Mon Sep 17 00:00:00 2001 From: Baz Date: Fri, 5 Aug 2022 21:46:11 +0300 Subject: [PATCH 18/25] updated releaseStage to beta (#15357) --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a285e9a4bc2e..54661564d6f5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -283,7 +283,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk icon: freshdesk.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Freshsales sourceDefinitionId: eca08d79-7b92-4065-b7f3-79c14836ebe7 dockerRepository: airbyte/source-freshsales From 9fa4b5ba3adfd905a0f0a9f274b401ca4ab770fa Mon Sep 17 00:00:00 2001 From: Baz Date: Fri, 5 Aug 2022 21:46:44 +0300 Subject: [PATCH 19/25] docs updated (#15356) --- docs/integrations/sources/freshdesk.md | 71 ++++++++++++++++++-------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md index b8941e099bac..1f76f164e515 100644 --- a/docs/integrations/sources/freshdesk.md +++ b/docs/integrations/sources/freshdesk.md @@ -1,13 +1,61 @@ # Freshdesk -## Overview +This page guides you through the process of setting up the Freshdesk source connector. + +## Prerequisites + +* Freshdesk Account +* Domain URL +* Freshdesk API Key + +## Step 1: Set up Freshdesk + +### Get Domain URL + +You can find your domain URL by loggin into your account and check the URL in your browser, the domain url should look like: `https://myaccount.freshdesk.com/...`, where `myaccount.freshdesk.com` - is your domain URL. + +### Get Freshdesk API Key + +Follow the link to read more about [how to find your API key](https://support.freshdesk.com/support/solutions/articles/215517). You need the admin permissions to access the account settings. + + +## Step 2: Set up the Freshdesk connector in Airbyte + +**For Airbyte Cloud** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **Freshdesk** from the Source type dropdown and enter a name for this connector. +4. Enter your `Domain URL`. +5. Enter your `Freshdesk API Key`. +6. Choose the `Start Date` as the starting point for your data replication. +5. Click `Set up source`. + +**For Airbyte OSS:** + +1. Go to local Airbyte page. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **Freshdesk** from the Source type dropdown and enter a name for this connector. +4. Enter your `Domain URL`. +5. Enter your `Freshdesk API Key`. +6. Choose the `Start Date` as the starting point for your data replication. +5. Click `Set up source`. + +## Supported sync modes & Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| SSL connection | Yes | +| Namespaces | No | The Freshdesk supports full refresh and incremental sync. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. There are two types of incremental sync: * server level \(native\) - when API supports filter on specific columns that Airbyte use to track changes \(`updated_at`, `created_at`, etc\) * client level - when API doesn't support filter and Airbyte performs filtration on its side. -### Output schema +## Supported Streams Several output streams are available from this source: @@ -25,29 +73,10 @@ Several output streams are available from this source: If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) -### Features - -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | Yes | -| SSL connection | Yes | -| Namespaces | No | - ### Performance considerations The Freshdesk connector should not run into Freshdesk API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. -## Getting started - -### Requirements - -* Freshdesk Account -* Freshdesk API Key - -### Setup guide - -Please read [How to find your API key](https://support.freshdesk.com/support/solutions/articles/215517). ## Changelog From 41eac4b4166061335a6f3e1f0c9a2690c3f07dd1 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 5 Aug 2022 15:29:30 -0600 Subject: [PATCH 20/25] feat: new constructor that allows for specifying max number of buffers (#14546) --- .../destination/record_buffer/FileBuffer.java | 13 +++++++++++-- .../s3/parquet/ParquetSerializedBuffer.java | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FileBuffer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FileBuffer.java index b85c8c97279c..a07a97ae1b39 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FileBuffer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FileBuffer.java @@ -32,14 +32,23 @@ public class FileBuffer implements BufferStorage { public static final long MAX_TOTAL_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024; // mb // we limit number of stream being buffered simultaneously anyway (limit how many files are // stored/open for writing) - public static final int MAX_CONCURRENT_STREAM_IN_BUFFER = 10; + public static final int DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER = 10; private final String fileExtension; private File tempFile; private OutputStream outputStream; + private final int maxConcurrentStreams; public FileBuffer(final String fileExtension) { this.fileExtension = fileExtension; + this.maxConcurrentStreams = DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER; + tempFile = null; + outputStream = null; + } + + public FileBuffer(final String fileExtension, final int maxConcurrentStreams) { + this.fileExtension = fileExtension; + this.maxConcurrentStreams = maxConcurrentStreams; tempFile = null; outputStream = null; } @@ -94,7 +103,7 @@ public long getMaxPerStreamBufferSizeInBytes() { @Override public int getMaxConcurrentStreamsInBuffer() { - return MAX_CONCURRENT_STREAM_IN_BUFFER; + return maxConcurrentStreams; } } diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java index 9ec576078861..c9564128dec9 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java @@ -145,7 +145,7 @@ public long getMaxPerStreamBufferSizeInBytes() { @Override public int getMaxConcurrentStreamsInBuffer() { - return FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER; + return FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER; } @Override From 986ae4816a55eeb853cecdd54684d99ea5f608f5 Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 5 Aug 2022 14:34:51 -0700 Subject: [PATCH 21/25] Update spec.json (#15342) * Update copy in spec.json Co-authored-by: grishick Co-authored-by: Greg Solovyev Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 85 +++++++++---------- .../source-postgres-strict-encrypt/Dockerfile | 2 +- .../connectors/source-postgres/Dockerfile | 2 +- .../src/main/resources/spec.json | 38 ++++----- .../src/test/resources/expected_spec.json | 38 ++++----- docs/integrations/sources/postgres.md | 1 + 7 files changed, 84 insertions(+), 84 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 54661564d6f5..5bce1ef760cf 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -762,7 +762,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 0.4.43 + dockerImageTag: 0.4.44 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 71a2a52ee125..9cd10f6bb513 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7138,7 +7138,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:0.4.43" +- dockerImage: "airbyte/source-postgres:0.4.44" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -7167,13 +7167,14 @@ - "5432" order: 1 database: - title: "DB Name" + title: "Database Name" description: "Name of the database." type: "string" order: 2 schemas: title: "Schemas" - description: "The list of schemas to sync from. Defaults to user. Case sensitive." + description: "The list of schemas (case sensitive) to sync from. Defaults\ + \ to public." type: "array" items: type: "string" @@ -7183,8 +7184,8 @@ - "public" order: 3 username: - title: "User" - description: "Username to use to access the database." + title: "Username" + description: "Username to access the database." type: "string" order: 4 password: @@ -7196,9 +7197,9 @@ jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ - \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For\ - \ more information read about additional JDBC URL parameters." + \ by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more\ + \ information read about JDBC URL parameters." title: "JDBC URL Parameters (Advanced)" type: "string" order: 6 @@ -7210,19 +7211,18 @@ default: false order: 7 ssl_mode: - title: "SSL modes" - description: "SSL connection modes. \n disable - Chose this mode\ - \ to disable encryption of communication between Airbyte and source database\n\ - \ allow - Chose this mode to enable encryption only when required\ - \ by the source database\n prefer - Chose this mode to allow unencrypted\ - \ connection only if the source database does not support encryption\n\ - \ require - Chose this mode to always require encryption. If the\ - \ source database server does not support encryption, connection will\ - \ fail\n verify-ca - Chose this mode to always require encryption\ - \ and to verify that the source database server has a valid SSL certificate\n\ - \ verify-full - This is the most secure mode. Chose this mode\ - \ to always require encryption and to verify the identity of the source\ - \ database server\n See more information -
  • disable - Disables\ + \ encryption of communication between Airbyte and source database
    • \n\ + \
  • allow - Enables encryption only when required by the source\ + \ database
    • \n
  • prefer - allows unencrypted connection only\ + \ if the source database does not support encryption
    • \n
  • require\ + \ - Always require encryption. If the source database server does not\ + \ support encryption, connection will fail
    • \n
  • verify-ca\ + \ - Always require encryption and verifies that the source database server\ + \ has a valid SSL certificate
    • \n
  • verify-full - This is\ + \ the most secure mode. Always require encryption and verifies the identity\ + \ of the source database server
    • \n Read more     in the docs." type: "object" order: 7 @@ -7302,9 +7302,9 @@ order: 1 client_key_password: type: "string" - title: "Client key password (Optional)" - description: "Password for keystorage. This field is optional. If\ - \ you do not add it - the password will be generated automatically." + title: "Client Key Password (Optional)" + description: "Password for keystorage. If you do not add it - the\ + \ password will be generated automatically." airbyte_secret: true order: 4 - title: "verify-full" @@ -7325,21 +7325,21 @@ order: 0 ca_certificate: type: "string" - title: "CA certificate" + title: "CA Certificate" description: "CA certificate" airbyte_secret: true multiline: true order: 1 client_certificate: type: "string" - title: "Client certificate" + title: "Client Certificate" description: "Client certificate" airbyte_secret: true multiline: true order: 2 client_key: type: "string" - title: "Client key" + title: "Client Key" description: "Client key" airbyte_secret: true multiline: true @@ -7347,14 +7347,14 @@ client_key_password: type: "string" title: "Client key password (Optional)" - description: "Password for keystorage. This field is optional. If\ - \ you do not add it - the password will be generated automatically." + description: "Password for keystorage. If you do not add it - the\ + \ password will be generated automatically." airbyte_secret: true order: 4 replication_method: type: "object" title: "Replication Method" - description: "Replication method to use for extracting data from the database." + description: "Replication method for extracting data from the database." order: 8 oneOf: - title: "Standard" @@ -7375,7 +7375,7 @@ \ to detect inserts, updates, and deletes. This needs to be configured\ \ on the source database itself. Only available on Postgres 10 and above.\ \ Read the Postgres Source docs for more information." + >docs." required: - "method" - "replication_slot" @@ -7391,12 +7391,11 @@ plugin: type: "string" title: "Plugin" - description: "A logical decoding plug-in installed on the PostgreSQL\ - \ server. `pgoutput` plug-in is used by default.\nIf replication\ + description: "A logical decoding plugin installed on the PostgreSQL\ + \ server. The `pgoutput` plugin is used by default. If the replication\ \ table contains a lot of big jsonb values it is recommended to\ - \ use `wal2json` plug-in. For more information about `wal2json`\ - \ plug-in read Select replication plugin." + \ use `wal2json` plugin. Read more about selecting replication plugins." enum: - "pgoutput" - "wal2json" @@ -7405,24 +7404,24 @@ replication_slot: type: "string" title: "Replication Slot" - description: "A plug-in logical replication slot. For more information\ - \ read about replication slots." order: 2 publication: type: "string" title: "Publication" - description: "A Postgres publication used for consuming changes. For\ - \ more information read about publications and replication identities." order: 3 initial_waiting_seconds: type: "integer" title: "Initial Waiting Time in Seconds (Advanced)" description: "The amount of time the connector will wait when it launches\ - \ to figure out whether there is new data to sync or not. Default\ - \ to 300 seconds. Valid range: 120 seconds to 1200 seconds. For\ - \ more information read about initial waiting time." default: 300 order: 4 diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 51c971e50f1b..10311b9df632 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.43 +LABEL io.airbyte.version=0.4.44 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 505b8a5ee566..570e873b0f09 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.43 +LABEL io.airbyte.version=0.4.44 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index e1761821f404..2436604cb9a2 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -23,14 +23,14 @@ "order": 1 }, "database": { - "title": "DB Name", + "title": "Database Name", "description": "Name of the database.", "type": "string", "order": 2 }, "schemas": { "title": "Schemas", - "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", + "description": "The list of schemas (case sensitive) to sync from. Defaults to public.", "type": "array", "items": { "type": "string" @@ -41,8 +41,8 @@ "order": 3 }, "username": { - "title": "User", - "description": "Username to use to access the database.", + "title": "Username", + "description": "Username to access the database.", "type": "string", "order": 4 }, @@ -54,7 +54,7 @@ "order": 5 }, "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about additional JDBC URL parameters.", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", "title": "JDBC URL Parameters (Advanced)", "type": "string", "order": 6 @@ -67,8 +67,8 @@ "order": 7 }, "ssl_mode": { - "title": "SSL modes", - "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and source database\n allow - Chose this mode to enable encryption only when required by the source database\n prefer - Chose this mode to allow unencrypted connection only if the source database does not support encryption\n require - Chose this mode to always require encryption. If the source database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the source database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the source database server\n See more information - in the docs.", + "title": "SSL Modes", + "description": "SSL connection modes. \n
    • disable - Disables encryption of communication between Airbyte and source database
      • \n
    • allow - Enables encryption only when required by the source database
      • \n
    • prefer - allows unencrypted connection only if the source database does not support encryption
      • \n
    • require - Always require encryption. If the source database server does not support encryption, connection will fail
      • \n
    • verify-ca - Always require encryption and verifies that the source database server has a valid SSL certificate
      • \n
    • verify-full - This is the most secure mode. Always require encryption and verifies the identity of the source database server
    \n Read more in the docs.", "type": "object", "order": 7, "oneOf": [ @@ -155,8 +155,8 @@ }, "client_key_password": { "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "title": "Client Key Password (Optional)", + "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, "order": 4 } @@ -182,7 +182,7 @@ }, "ca_certificate": { "type": "string", - "title": "CA certificate", + "title": "CA Certificate", "description": "CA certificate", "airbyte_secret": true, "multiline": true, @@ -190,7 +190,7 @@ }, "client_certificate": { "type": "string", - "title": "Client certificate", + "title": "Client Certificate", "description": "Client certificate", "airbyte_secret": true, "multiline": true, @@ -198,7 +198,7 @@ }, "client_key": { "type": "string", - "title": "Client key", + "title": "Client Key", "description": "Client key", "airbyte_secret": true, "multiline": true, @@ -207,7 +207,7 @@ "client_key_password": { "type": "string", "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, "order": 4 } @@ -218,7 +218,7 @@ "replication_method": { "type": "object", "title": "Replication Method", - "description": "Replication method to use for extracting data from the database.", + "description": "Replication method for extracting data from the database.", "order": 8, "oneOf": [ { @@ -237,7 +237,7 @@ }, { "title": "Logical Replication (CDC)", - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the Postgres Source docs for more information.", + "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the docs.", "required": ["method", "replication_slot", "publication"], "properties": { "method": { @@ -250,7 +250,7 @@ "plugin": { "type": "string", "title": "Plugin", - "description": "A logical decoding plug-in installed on the PostgreSQL server. `pgoutput` plug-in is used by default.\nIf replication table contains a lot of big jsonb values it is recommended to use `wal2json` plug-in. For more information about `wal2json` plug-in read Select replication plugin.", + "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], "default": "pgoutput", "order": 1 @@ -258,19 +258,19 @@ "replication_slot": { "type": "string", "title": "Replication Slot", - "description": "A plug-in logical replication slot. For more information read about replication slots.", + "description": "A plugin logical replication slot. Read about replication slots.", "order": 2 }, "publication": { "type": "string", "title": "Publication", - "description": "A Postgres publication used for consuming changes. For more information read about publications and replication identities.", + "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", "order": 3 }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to figure out whether there is new data to sync or not. Default to 300 seconds. Valid range: 120 seconds to 1200 seconds. For more information read about initial waiting time.", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", "default": 300, "order": 4, "min": 120, diff --git a/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json index cb8c04865a86..3b4d07b3c636 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json @@ -23,14 +23,14 @@ "order": 1 }, "database": { - "title": "DB Name", + "title": "Database Name", "description": "Name of the database.", "type": "string", "order": 2 }, "schemas": { "title": "Schemas", - "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", + "description": "The list of schemas (case sensitive) to sync from. Defaults to public.", "type": "array", "items": { "type": "string" @@ -41,8 +41,8 @@ "order": 3 }, "username": { - "title": "User", - "description": "Username to use to access the database.", + "title": "Username", + "description": "Username to access the database.", "type": "string", "order": 4 }, @@ -54,14 +54,14 @@ "order": 5 }, "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about additional JDBC URL parameters.", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", "title": "JDBC URL Parameters (Advanced)", "type": "string", "order": 6 }, "ssl_mode": { - "title": "SSL modes", - "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and source database\n allow - Chose this mode to enable encryption only when required by the source database\n prefer - Chose this mode to allow unencrypted connection only if the source database does not support encryption\n require - Chose this mode to always require encryption. If the source database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the source database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the source database server\n See more information - in the docs.", + "title": "SSL Modes", + "description": "SSL connection modes. \n
    • disable - Disables encryption of communication between Airbyte and source database
      • \n
    • allow - Enables encryption only when required by the source database
      • \n
    • prefer - allows unencrypted connection only if the source database does not support encryption
      • \n
    • require - Always require encryption. If the source database server does not support encryption, connection will fail
      • \n
    • verify-ca - Always require encryption and verifies that the source database server has a valid SSL certificate
      • \n
    • verify-full - This is the most secure mode. Always require encryption and verifies the identity of the source database server
    \n Read more in the docs.", "type": "object", "order": 7, "oneOf": [ @@ -133,8 +133,8 @@ }, "client_key_password": { "type": "string", - "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "title": "Client Key Password (Optional)", + "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, "order": 4 } @@ -160,7 +160,7 @@ }, "ca_certificate": { "type": "string", - "title": "CA certificate", + "title": "CA Certificate", "description": "CA certificate", "airbyte_secret": true, "multiline": true, @@ -168,7 +168,7 @@ }, "client_certificate": { "type": "string", - "title": "Client certificate", + "title": "Client Certificate", "description": "Client certificate", "airbyte_secret": true, "multiline": true, @@ -176,7 +176,7 @@ }, "client_key": { "type": "string", - "title": "Client key", + "title": "Client Key", "description": "Client key", "airbyte_secret": true, "multiline": true, @@ -185,7 +185,7 @@ "client_key_password": { "type": "string", "title": "Client key password (Optional)", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, "order": 4 } @@ -196,7 +196,7 @@ "replication_method": { "type": "object", "title": "Replication Method", - "description": "Replication method to use for extracting data from the database.", + "description": "Replication method for extracting data from the database.", "order": 8, "oneOf": [ { @@ -215,7 +215,7 @@ }, { "title": "Logical Replication (CDC)", - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the Postgres Source docs for more information.", + "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the docs.", "required": ["method", "replication_slot", "publication"], "properties": { "method": { @@ -228,7 +228,7 @@ "plugin": { "type": "string", "title": "Plugin", - "description": "A logical decoding plug-in installed on the PostgreSQL server. `pgoutput` plug-in is used by default.\nIf replication table contains a lot of big jsonb values it is recommended to use `wal2json` plug-in. For more information about `wal2json` plug-in read Select replication plugin.", + "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], "default": "pgoutput", "order": 1 @@ -236,19 +236,19 @@ "replication_slot": { "type": "string", "title": "Replication Slot", - "description": "A plug-in logical replication slot. For more information read about replication slots.", + "description": "A plugin logical replication slot. Read about replication slots.", "order": 2 }, "publication": { "type": "string", "title": "Publication", - "description": "A Postgres publication used for consuming changes. For more information read about publications and replication identities.", + "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", "order": 3 }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to figure out whether there is new data to sync or not. Default to 300 seconds. Valid range: 120 seconds to 1200 seconds. For more information read about initial waiting time.", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", "default": 300, "order": 4, "min": 120, diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 9d61d4b4a93a..5541e813fbda 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -371,6 +371,7 @@ Possible solutions include: | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | | 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | | 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | | 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | From bd3110077490ade90a62a376ef674ac840db4f05 Mon Sep 17 00:00:00 2001 From: Brian Lai <51336873+brianjlai@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:39:27 -0400 Subject: [PATCH 22/25] initial first pass converting every component to dataclasses (#15189) * initial first pass converting every component to dataclasses * replace the hackier options pass through * get rid of the hackier way * fix issues w/ type hints by making options required and lots of fixes to the language to fix compatability for dataclasses * add dataclasses-jsonschema to setup * fix oauth authenticator to avoid dataclass name collisions * fix spacing for CI tests * remove property from oauth and fix a interpolation bug * pr feedback and cleaning up the code a bit, attempt at avoiding renaming * fix templates and bugs surfaced during greenhouse spec testing * fix tests * fix missing options in some declarative components * fix tests related to pulling latest master * fix issue w/ passing state, slice, and token to subcomponents * switch name back to get_access_token() since no name collision anymore --- .../sources/declarative/auth/oauth.py | 170 ++++++--------- .../sources/declarative/auth/token.py | 97 ++++----- .../sources/declarative/checks/__init__.py | 5 + .../declarative/checks/check_stream.py | 22 +- .../sources/declarative/create_partial.py | 17 +- .../sources/declarative/datetime/__init__.py | 4 + .../declarative/datetime/min_max_datetime.py | 72 ++++--- .../sources/declarative/declarative_stream.py | 88 ++++---- .../sources/declarative/decoders/__init__.py | 7 +- .../sources/declarative/decoders/decoder.py | 2 + .../declarative/decoders/json_decoder.py | 4 + .../declarative/extractors/__init__.py | 7 + .../declarative/extractors/http_selector.py | 2 + .../sources/declarative/extractors/jello.py | 36 ++-- .../declarative/extractors/record_filter.py | 25 ++- .../declarative/extractors/record_selector.py | 28 +-- .../declarative/interpolation/__init__.py | 8 +- .../interpolation/interpolated_boolean.py | 25 ++- .../interpolation/interpolated_mapping.py | 29 ++- .../interpolation/interpolated_string.py | 32 +-- .../declarative/requesters/__init__.py | 8 +- .../requesters/error_handlers/__init__.py | 8 + .../backoff_strategies/__init__.py | 18 ++ .../constant_backoff_strategy.py | 16 +- .../exponential_backoff_strategy.py | 16 +- .../wait_time_from_header_backoff_strategy.py | 23 +- ...until_time_from_header_backoff_strategy.py | 34 +-- .../error_handlers/backoff_strategy.py | 2 + .../error_handlers/composite_error_handler.py | 24 ++- .../error_handlers/default_error_handler.py | 63 +++--- .../error_handlers/error_handler.py | 2 + .../error_handlers/http_response_filter.py | 53 ++--- .../declarative/requesters/http_requester.py | 107 +++++----- .../requesters/paginators/__init__.py | 9 +- .../requesters/paginators/limit_paginator.py | 98 ++++----- .../requesters/paginators/no_pagination.py | 12 +- .../requesters/paginators/paginator.py | 2 + .../paginators/strategies/__init__.py | 6 + .../strategies/cursor_pagination_strategy.py | 51 +++-- .../paginators/strategies/offset_increment.py | 21 +- .../paginators/strategies/page_increment.py | 21 +- .../{ => strategies}/pagination_strategy.py | 5 +- .../declarative/requesters/request_option.py | 46 ++-- .../requesters/request_options/__init__.py | 7 + .../interpolated_request_input_provider.py | 0 .../interpolated_request_options_provider.py | 76 +++---- .../request_options_provider.py | 10 +- .../declarative/requesters/requester.py | 8 +- .../declarative/retrievers/__init__.py | 5 + .../declarative/retrievers/retriever.py | 2 + .../retrievers/simple_retriever.py | 132 ++++++------ .../sources/declarative/schema/__init__.py | 5 + .../sources/declarative/schema/json_schema.py | 35 +-- .../declarative/schema/schema_loader.py | 2 + .../declarative/stream_slicers/__init__.py | 9 + .../cartesian_product_stream_slicer.py | 58 +++-- .../stream_slicers/datetime_stream_slicer.py | 141 ++++++------- .../stream_slicers/list_stream_slicer.py | 69 +++--- .../stream_slicers/single_slice.py | 16 +- .../stream_slicers/stream_slicer.py | 2 + .../stream_slicers/substream_slicer.py | 38 ++-- .../declarative/transformations/add_fields.py | 50 +++-- .../transformations/remove_fields.py | 19 +- .../transformations/transformation.py | 2 + .../requests_native_auth/abstract_oauth.py | 56 ++--- .../http/requests_native_auth/oauth.py | 92 +++----- .../python/reference_docs/_source/conf.py | 5 +- airbyte-cdk/python/setup.py | 1 + .../sources/declarative/auth/test_oauth.py | 11 +- .../declarative/auth/test_token_auth.py | 10 +- .../declarative/checks/test_check_stream.py | 2 +- .../datetime/test_min_max_datetime.py | 26 ++- .../declarative/extractors/test_jello.py | 6 +- .../extractors/test_record_filter.py | 9 +- .../extractors/test_record_selector.py | 16 +- .../test_interpolated_boolean.py | 4 +- .../test_interpolated_mapping.py | 2 +- .../interpolation/test_interpolated_string.py | 1 + .../declarative/iterators/test_only_once.py | 2 +- .../test_exponential_backoff.py | 7 + .../test_composite_error_handler.py | 4 +- .../test_default_error_handler.py | 18 +- .../test_cursor_pagination_strategy.py | 12 +- .../paginators/test_limit_paginator.py | 56 +++-- .../paginators/test_no_paginator.py | 2 +- .../paginators/test_offset_increment.py | 2 +- .../paginators/test_page_increment.py | 2 +- .../paginators/test_request_option.py | 6 +- ...t_interpolated_request_options_provider.py | 14 +- .../requesters/test_http_requester.py | 15 +- ...est_interpolated_request_input_provider.py | 2 +- .../retrievers/test_simple_retriever.py | 77 ++++--- .../test_cartesian_product_stream_slicer.py | 85 +++++--- .../test_datetime_stream_slicer.py | 110 +++++----- ...t_slicer.py => test_list_stream_slicer.py} | 42 ++-- .../stream_slicers/test_substream_slicer.py | 103 ++++++--- .../declarative/test_create_partial.py | 15 +- .../declarative/test_declarative_stream.py | 3 +- .../sources/declarative/test_factory.py | 199 ++++++++++-------- .../transformations/test_add_fields.py | 4 +- .../transformations/test_remove_fields.py | 2 +- .../test_requests_native_auth.py | 2 +- .../{{snakeCase name}}.yaml.hbs | 9 +- 103 files changed, 1712 insertions(+), 1333 deletions(-) rename airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/{ => strategies}/pagination_strategy.py (80%) rename airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/{ => request_options}/interpolated_request_input_provider.py (100%) rename airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/{test_list_slicer.py => test_list_stream_slicer.py} (61%) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py index d20864f47eb1..ff9d5ef8b104 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py @@ -2,146 +2,94 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass, field from typing import Any, List, Mapping, Optional, Union import pendulum from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator +from dataclasses_jsonschema import JsonSchemaMixin -class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator): +@dataclass +class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, JsonSchemaMixin): """ Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials based on a declarative connector configuration file. Credentials can be defined explicitly or via interpolation at runtime. The generated access token is attached to each request via the Authorization header. + + Attributes: + token_refresh_endpoint (Union[InterpolatedString, str]): The endpoint to refresh the access token + client_id (Union[InterpolatedString, str]): The client id + client_secret (Union[InterpolatedString, str]): Client secret + refresh_token (Union[InterpolatedString, str]): The token used to refresh the access token + access_token_name (Union[InterpolatedString, str]): THe field to extract access token from in the response + expires_in_name (Union[InterpolatedString, str]): The field to extract expires_in from in the response + config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec + scopes (Optional[List[str]]): The scopes to request + token_expiry_date (Optional[Union[InterpolatedString, str]]): The access token expiration date + refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request """ - def __init__( - self, - token_refresh_endpoint: Union[InterpolatedString, str], - client_id: Union[InterpolatedString, str], - client_secret: Union[InterpolatedString, str], - refresh_token: Union[InterpolatedString, str], - config: Mapping[str, Any], - scopes: Optional[List[str]] = None, - token_expiry_date: Optional[Union[InterpolatedString, str]] = None, - access_token_name: Union[InterpolatedString, str] = "access_token", - expires_in_name: Union[InterpolatedString, str] = "expires_in", - refresh_request_body: Optional[Mapping[str, Any]] = None, - **options: Optional[Mapping[str, Any]], - ): - """ - :param token_refresh_endpoint: The endpoint to refresh the access token - :param client_id: The client id - :param client_secret: Client secret - :param refresh_token: The token used to refresh the access token - :param config: The user-provided configuration as specified by the source's spec - :param scopes: The scopes to request - :param token_expiry_date: The access token expiration date - :param access_token_name: THe field to extract access token from in the response - :param expires_in_name:The field to extract expires_in from in the response - :param refresh_request_body: The request body to send in the refresh request - :param options: Additional runtime parameters to be used for string interpolation - """ - self.config = config - self.token_refresh_endpoint = InterpolatedString.create(token_refresh_endpoint, options=options) - self.client_secret = InterpolatedString.create(client_secret, options=options) - self.client_id = InterpolatedString.create(client_id, options=options) - self.refresh_token = InterpolatedString.create(refresh_token, options=options) - self.scopes = scopes - self.access_token_name = InterpolatedString.create(access_token_name, options=options) - self.expires_in_name = InterpolatedString.create(expires_in_name, options=options) - self.refresh_request_body = InterpolatedMapping(refresh_request_body or {}, options=options) - - self.token_expiry_date = ( - pendulum.parse(InterpolatedString.create(token_expiry_date, options=options).eval(self.config)) - if token_expiry_date + token_refresh_endpoint: Union[InterpolatedString, str] + client_id: Union[InterpolatedString, str] + client_secret: Union[InterpolatedString, str] + refresh_token: Union[InterpolatedString, str] + config: Mapping[str, Any] + options: InitVar[Mapping[str, Any]] + scopes: Optional[List[str]] = None + token_expiry_date: Optional[Union[InterpolatedString, str]] = None + _token_expiry_date: pendulum.DateTime = field(init=False, repr=False) + access_token_name: Union[InterpolatedString, str] = "access_token" + expires_in_name: Union[InterpolatedString, str] = "expires_in" + refresh_request_body: Optional[Mapping[str, Any]] = None + + def __post_init__(self, options: Mapping[str, Any]): + self.token_refresh_endpoint = InterpolatedString.create(self.token_refresh_endpoint, options=options) + self.client_id = InterpolatedString.create(self.client_id, options=options) + self.client_secret = InterpolatedString.create(self.client_secret, options=options) + self.refresh_token = InterpolatedString.create(self.refresh_token, options=options) + self.access_token_name = InterpolatedString.create(self.access_token_name, options=options) + self.expires_in_name = InterpolatedString.create(self.expires_in_name, options=options) + self._refresh_request_body = InterpolatedMapping(self.refresh_request_body or {}, options=options) + self._token_expiry_date = ( + pendulum.parse(InterpolatedString.create(self.token_expiry_date, options=options).eval(self.config)) + if self.token_expiry_date else pendulum.now().subtract(days=1) ) - self.access_token = None - - @property - def config(self) -> Mapping[str, Any]: - return self._config - - @config.setter - def config(self, value: Mapping[str, Any]): - self._config = value - - @property - def token_refresh_endpoint(self) -> InterpolatedString: - get_some = self._token_refresh_endpoint.eval(self.config) - return get_some + self._access_token = None - @token_refresh_endpoint.setter - def token_refresh_endpoint(self, value: InterpolatedString): - self._token_refresh_endpoint = value + def get_token_refresh_endpoint(self) -> str: + return self.token_refresh_endpoint.eval(self.config) - @property - def client_id(self) -> InterpolatedString: - return self._client_id.eval(self.config) - - @client_id.setter - def client_id(self, value: InterpolatedString): - self._client_id = value + def get_client_id(self) -> str: + return self.client_id.eval(self.config) - @property - def client_secret(self) -> InterpolatedString: - return self._client_secret.eval(self.config) + def get_client_secret(self) -> str: + return self.client_secret.eval(self.config) - @client_secret.setter - def client_secret(self, value: InterpolatedString): - self._client_secret = value + def get_refresh_token(self) -> str: + return self.refresh_token.eval(self.config) - @property - def refresh_token(self) -> InterpolatedString: - return self._refresh_token.eval(self.config) + def get_scopes(self) -> [str]: + return self.scopes - @refresh_token.setter - def refresh_token(self, value: InterpolatedString): - self._refresh_token = value + def get_access_token_name(self) -> InterpolatedString: + return self.access_token_name.eval(self.config) - @property - def scopes(self) -> [str]: - return self._scopes + def get_expires_in_name(self) -> InterpolatedString: + return self.expires_in_name.eval(self.config) - @scopes.setter - def scopes(self, value: [str]): - self._scopes = value + def get_refresh_request_body(self) -> Mapping[str, Any]: + return self._refresh_request_body.eval(self.config) - @property - def token_expiry_date(self) -> pendulum.DateTime: + def get_token_expiry_date(self) -> pendulum.DateTime: return self._token_expiry_date - @token_expiry_date.setter - def token_expiry_date(self, value: pendulum.DateTime): + def set_token_expiry_date(self, value: pendulum.DateTime): self._token_expiry_date = value - @property - def access_token_name(self) -> InterpolatedString: - return self._access_token_name.eval(self.config) - - @access_token_name.setter - def access_token_name(self, value: InterpolatedString): - self._access_token_name = value - - @property - def expires_in_name(self) -> InterpolatedString: - return self._expires_in_name.eval(self.config) - - @expires_in_name.setter - def expires_in_name(self, value: InterpolatedString): - self._expires_in_name = value - - @property - def refresh_request_body(self) -> InterpolatedMapping: - return self._refresh_request_body.eval(self.config) - - @refresh_request_body.setter - def refresh_request_body(self, value: InterpolatedMapping): - self._refresh_request_body = value - @property def access_token(self) -> str: return self._access_token diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py index 30520b03d971..04790ae9e303 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -3,14 +3,17 @@ # import base64 -from typing import Any, Mapping, Optional, Union +from dataclasses import InitVar, dataclass +from typing import Any, Mapping, Union from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Config from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator +from dataclasses_jsonschema import JsonSchemaMixin -class ApiKeyAuthenticator(AbstractHeaderAuthenticator): +@dataclass +class ApiKeyAuthenticator(AbstractHeaderAuthenticator, JsonSchemaMixin): """ ApiKeyAuth sets a request header on the HTTP requests sent. @@ -22,50 +25,51 @@ class ApiKeyAuthenticator(AbstractHeaderAuthenticator): will result in the following header set on the HTTP request `"Authorization": "Bearer hello"` + Attributes: + header (Union[InterpolatedString, str]): Header key to set on the HTTP requests + api_token (Union[InterpolatedString, str]): Header value to set on the HTTP requests + config (Config): The user-provided configuration as specified by the source's spec + options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - def __init__( - self, - header: Union[InterpolatedString, str], - token: Union[InterpolatedString, str], - config: Config, - **options: Optional[Mapping[str, Any]], - ): - """ - :param header: Header key to set on the HTTP requests - :param token: Header value to set on the HTTP requests - :param config: The user-provided configuration as specified by the source's spec - :param options: Additional runtime parameters to be used for string interpolation - """ - self._header = InterpolatedString.create(header, options=options) - self._token = InterpolatedString.create(token, options=options) - self._config = config + header: Union[InterpolatedString, str] + api_token: Union[InterpolatedString, str] + config: Config + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): + self._header = InterpolatedString.create(self.header, options=options) + self._token = InterpolatedString.create(self.api_token, options=options) @property def auth_header(self) -> str: - return self._header.eval(self._config) + return self._header.eval(self.config) @property def token(self) -> str: - return self._token.eval(self._config) + return self._token.eval(self.config) -class BearerAuthenticator(AbstractHeaderAuthenticator): +@dataclass +class BearerAuthenticator(AbstractHeaderAuthenticator, JsonSchemaMixin): """ Authenticator that sets the Authorization header on the HTTP requests sent. The header is of the form: `"Authorization": "Bearer "` + + Attributes: + api_token (Union[InterpolatedString, str]): The bearer token + config (Config): The user-provided configuration as specified by the source's spec + options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - def __init__(self, token: Union[InterpolatedString, str], config: Config, **options: Optional[Mapping[str, Any]]): - """ - :param token: The bearer token - :param config: The user-provided configuration as specified by the source's spec - :param options: Additional runtime parameters to be used for string interpolation - """ - self._token = InterpolatedString.create(token, options=options) - self._config = config + api_token: Union[InterpolatedString, str] + config: Config + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): + self._token = InterpolatedString.create(self.api_token, options=options) @property def auth_header(self) -> str: @@ -73,9 +77,10 @@ def auth_header(self) -> str: @property def token(self) -> str: - return f"Bearer {self._token.eval(self._config)}" + return f"Bearer {self._token.eval(self.config)}" +@dataclass class BasicHttpAuthenticator(AbstractHeaderAuthenticator): """ Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using bas64 @@ -83,24 +88,22 @@ class BasicHttpAuthenticator(AbstractHeaderAuthenticator): The header is of the form `"Authorization": "Basic "` + + Attributes: + username (Union[InterpolatedString, str]): The username + config (Config): The user-provided configuration as specified by the source's spec + password (Union[InterpolatedString, str]): The password + options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - def __init__( - self, - username: Union[InterpolatedString, str], - config: Config, - password: Union[InterpolatedString, str] = "", - **options: Optional[Mapping[str, Any]], - ): - """ - :param username: The username - :param config: The user-provided configuration as specified by the source's spec - :param password: The password - :param options: Additional runtime parameters to be used for string interpolation - """ - self._username = InterpolatedString.create(username, options=options) - self._password = InterpolatedString.create(password, options=options) - self._config = config + username: Union[InterpolatedString, str] + config: Config + options: InitVar[Mapping[str, Any]] + password: Union[InterpolatedString, str] = "" + + def __post_init__(self, options): + self._username = InterpolatedString.create(self.username, options=options) + self._password = InterpolatedString.create(self.password, options=options) @property def auth_header(self) -> str: @@ -108,6 +111,6 @@ def auth_header(self) -> str: @property def token(self) -> str: - auth_string = f"{self._username.eval(self._config)}:{self._password.eval(self._config)}".encode("utf8") + auth_string = f"{self._username.eval(self.config)}:{self._password.eval(self.config)}".encode("utf8") b64_encoded = base64.b64encode(auth_string).decode("utf8") return f"Basic {b64_encoded}" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/__init__.py index 1100c1c58cf5..fb6665d94625 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/__init__.py @@ -1,3 +1,8 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream +from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker + +__all__ = ["CheckStream", "ConnectionChecker"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py index 47db5130ad96..decf9fefc862 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py @@ -3,24 +3,28 @@ # import logging -from typing import Any, List, Mapping, Optional, Tuple +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Tuple from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker from airbyte_cdk.sources.source import Source +from dataclasses_jsonschema import JsonSchemaMixin -class CheckStream(ConnectionChecker): +@dataclass +class CheckStream(ConnectionChecker, JsonSchemaMixin): """ Checks the connections by trying to read records from one or many of the streams selected by the developer + + Attributes: + stream_name (List[str]): name of streams to read records from """ - def __init__(self, stream_names: List[str], **options: Optional[Mapping[str, Any]]): - """ - :param stream_names: name of streams to read records from - :param options: Additional runtime parameters to be used for string interpolation - """ - self._stream_names = set(stream_names) + stream_names: List[str] + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): self._options = options def check_connection(self, source: Source, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: @@ -28,7 +32,7 @@ def check_connection(self, source: Source, logger: logging.Logger, config: Mappi stream_name_to_stream = {s.name: s for s in streams} if len(streams) == 0: return False, f"No streams to connect to from source {source}" - for stream_name in self._stream_names: + for stream_name in self.stream_names: if stream_name in stream_name_to_stream.keys(): stream = stream_name_to_stream[stream_name] try: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py index c4b9f4ac5619..c941153f3f84 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py @@ -41,8 +41,17 @@ def newfunc(*fargs, **fkeywords): kwargs_to_pass_down = _get_kwargs_to_pass_to_func(func, options) all_keywords_to_pass_down = _get_kwargs_to_pass_to_func(func, all_keywords) + + # options is required as part of creation of all declarative components + dynamic_args = {**all_keywords_to_pass_down, **kwargs_to_pass_down} + if "options" not in dynamic_args: + dynamic_args["options"] = {} + else: + # Handles the case where kwarg options and keyword $options both exist. We should merge both sets of options + # before creating the component + dynamic_args["options"] = {**all_keywords_to_pass_down["options"], **kwargs_to_pass_down["options"]} try: - ret = func(*args, *fargs, **{**all_keywords_to_pass_down, **kwargs_to_pass_down}) + ret = func(*args, *fargs, **dynamic_args) except TypeError as e: raise Exception(f"failed to create object of type {func} because {e}") return ret @@ -54,12 +63,14 @@ def newfunc(*fargs, **fkeywords): return newfunc -def _get_kwargs_to_pass_to_func(func, kwargs): +def _get_kwargs_to_pass_to_func(func, options): argspec = inspect.getfullargspec(func) kwargs_to_pass_down = set(argspec.kwonlyargs) args_to_pass_down = set(argspec.args) all_args = args_to_pass_down.union(kwargs_to_pass_down) - kwargs_to_pass_down = {k: v for k, v in kwargs.items() if k in all_args} + kwargs_to_pass_down = {k: v for k, v in options.items() if k in all_args} + if "options" in all_args: + kwargs_to_pass_down["options"] = options return kwargs_to_pass_down diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/__init__.py index 1100c1c58cf5..3832a103f682 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/__init__.py @@ -1,3 +1,7 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime + +__all__ = ["MinMaxDatetime"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py index 8f90766a5e25..d58ca36631c7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py @@ -3,38 +3,43 @@ # import datetime as dt -from typing import Any, Mapping, Optional, Union +from dataclasses import InitVar, dataclass, field +from typing import Any, Mapping, Union from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from dataclasses_jsonschema import JsonSchemaMixin -class MinMaxDatetime: +@dataclass +class MinMaxDatetime(JsonSchemaMixin): """ Compares the provided date against optional minimum or maximum times. If date is earlier than min_date, then min_date is returned. If date is greater than max_date, then max_date is returned. If neither, the input date is returned. + + Attributes: + datetime (Union[InterpolatedString, str]): InterpolatedString or string representing the datetime in the format specified by `datetime_format` + datetime_format (str): Format of the datetime passed as argument + min_datetime (Union[InterpolatedString, str]): Represents the minimum allowed datetime value. + max_datetime (Union[InterpolatedString, str]): Represents the maximum allowed datetime value. """ - def __init__( - self, - datetime: Union[InterpolatedString, str], - datetime_format: str = "", - min_datetime: Union[InterpolatedString, str] = "", - max_datetime: Union[InterpolatedString, str] = "", - **options: Optional[Mapping[str, Any]], - ): - """ - :param datetime: InterpolatedString or string representing the datetime in the format specified by `datetime_format` - :param datetime_format: Format of the datetime passed as argument - :param min_datetime: InterpolatedString or string representing the min datetime - :param max_datetime: InterpolatedString or string representing the max datetime - :param options: Additional runtime parameters to be used for string interpolation - """ - self._datetime_interpolator = InterpolatedString.create(datetime, options=options) - self._datetime_format = datetime_format + datetime: Union[InterpolatedString, str] + options: InitVar[Mapping[str, Any]] + # datetime_format is a unique case where we inherit it from the parent if it is not specified before using the default value + # which is why we need dedicated getter/setter methods and private dataclass field + datetime_format: str = "" + _datetime_format: str = field(init=False, repr=False, default="") + min_datetime: Union[InterpolatedString, str] = "" + max_datetime: Union[InterpolatedString, str] = "" + + def __post_init__(self, options: Mapping[str, Any]): + self.datetime = InterpolatedString.create(self.datetime, options=options or {}) + self.timezone = dt.timezone.utc + self.min_datetime = InterpolatedString.create(self.min_datetime, options=options) if self.min_datetime else None + self.max_datetime = InterpolatedString.create(self.max_datetime, options=options) if self.max_datetime else None + self._timezone = dt.timezone.utc - self._min_datetime_interpolator = InterpolatedString.create(min_datetime, options=options) if min_datetime else None - self._max_datetime_interpolator = InterpolatedString.create(max_datetime, options=options) if max_datetime else None def get_datetime(self, config, **additional_options) -> dt.datetime: """ @@ -48,19 +53,17 @@ def get_datetime(self, config, **additional_options) -> dt.datetime: if not datetime_format: datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z" - time = dt.datetime.strptime(str(self._datetime_interpolator.eval(config, **additional_options)), datetime_format).replace( - tzinfo=self._timezone - ) + time = dt.datetime.strptime(str(self.datetime.eval(config, **additional_options)), datetime_format).replace(tzinfo=self._timezone) - if self._min_datetime_interpolator: - min_time = dt.datetime.strptime( - str(self._min_datetime_interpolator.eval(config, **additional_options)), datetime_format - ).replace(tzinfo=self._timezone) + if self.min_datetime: + min_time = dt.datetime.strptime(str(self.min_datetime.eval(config, **additional_options)), datetime_format).replace( + tzinfo=self._timezone + ) time = max(time, min_time) - if self._max_datetime_interpolator: - max_time = dt.datetime.strptime( - str(self._max_datetime_interpolator.eval(config, **additional_options)), datetime_format - ).replace(tzinfo=self._timezone) + if self.max_datetime: + max_time = dt.datetime.strptime(str(self.max_datetime.eval(config, **additional_options)), datetime_format).replace( + tzinfo=self._timezone + ) time = min(time, max_time) return time @@ -72,4 +75,7 @@ def datetime_format(self) -> str: @datetime_format.setter def datetime_format(self, value: str): """Setter for the datetime format""" - self._datetime_format = value + # Covers the case where datetime_format is not provided in the constructor, which causes the property object + # to be set which we need to avoid doing + if not isinstance(value, property): + self._datetime_format = value diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py index fce2e20b8e58..feae3fa4d51a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py @@ -2,7 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # - +from dataclasses import InitVar, dataclass, field from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from airbyte_cdk.models import SyncMode @@ -11,46 +11,51 @@ from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, StreamSlice from airbyte_cdk.sources.streams.core import Stream +from dataclasses_jsonschema import JsonSchemaMixin -class DeclarativeStream(Stream): +@dataclass +class DeclarativeStream(Stream, JsonSchemaMixin): """ DeclarativeStream is a Stream that delegates most of its logic to its schema_load and retriever + + Attributes: + stream_name (str): stream name + stream_primary_key (Optional[Union[str, List[str], List[List[str]]]]): the primary key of the stream + schema_loader (SchemaLoader): The schema loader + retriever (Retriever): The retriever + config (Config): The user-provided configuration as specified by the source's spec + stream_cursor_field (Optional[List[str]]): The cursor field + transformations (List[RecordTransformation]): A list of transformations to be applied to each output record in the + stream. Transformations are applied in the order in which they are defined. + checkpoint_interval (Optional[int]): How often the stream will checkpoint state (i.e: emit a STATE message) """ - def __init__( - self, - name: str, - primary_key, - schema_loader: SchemaLoader, - retriever: Retriever, - config: Config, - cursor_field: Optional[List[str]] = None, - transformations: List[RecordTransformation] = None, - checkpoint_interval: Optional[int] = None, - ): - """ - :param name: stream name - :param primary_key: the primary key of the stream - :param schema_loader: The schema loader - :param retriever: The retriever - :param cursor_field: The cursor field - :param transformations: A list of transformations to be applied to each output record in the stream. Transformations are applied - in the order in which they are defined. - """ - self._name = name - self._config = config - self._primary_key = primary_key - self._cursor_field = cursor_field or [] - self._schema_loader = schema_loader - self._retriever = retriever - self._transformations = transformations or [] - self._checkpoint_interval = checkpoint_interval + schema_loader: SchemaLoader + retriever: Retriever + config: Config + options: InitVar[Mapping[str, Any]] + name: str + _name: str = field(init=False, repr=False) + primary_key: Optional[Union[str, List[str], List[List[str]]]] + _primary_key: str = field(init=False, repr=False) + stream_cursor_field: Optional[List[str]] = None + transformations: List[RecordTransformation] = None + checkpoint_interval: Optional[int] = None + + def __post_init__(self, options: Mapping[str, Any]): + self.stream_cursor_field = self.stream_cursor_field or [] + self.transformations = self.transformations or [] @property def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: return self._primary_key + @primary_key.setter + def primary_key(self, value: str) -> None: + if not isinstance(value, property): + self._primary_key = value + @property def name(self) -> str: """ @@ -58,6 +63,11 @@ def name(self) -> str: """ return self._name + @name.setter + def name(self, value: str) -> None: + if not isinstance(value, property): + self._name = value + @property def state_checkpoint_interval(self) -> Optional[int]: """ @@ -70,16 +80,16 @@ def state_checkpoint_interval(self) -> Optional[int]: ascending order with respect to the cursor field. This can happen if the source does not support reading records in ascending order of created_at date (or whatever the cursor is). In those cases, state must only be saved once the full stream has been read. """ - return self._checkpoint_interval + return self.checkpoint_interval @property def state(self) -> MutableMapping[str, Any]: - return self._retriever.state + return self.retriever.state @state.setter def state(self, value: MutableMapping[str, Any]): """State setter, accept state serialized by state getter.""" - self._retriever.state = value + self.retriever.state = value def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): return self.state @@ -90,7 +100,7 @@ def cursor_field(self) -> Union[str, List[str]]: Override to return the default cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. :return: The name of the field used as a cursor. If the cursor is nested, return an array consisting of the path to the cursor. """ - return self._cursor_field + return self.stream_cursor_field def read_records( self, @@ -99,12 +109,12 @@ def read_records( stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: - for record in self._retriever.read_records(sync_mode, cursor_field, stream_slice, stream_state): - yield self._apply_transformations(record, self._config, stream_slice) + for record in self.retriever.read_records(sync_mode, cursor_field, stream_slice, stream_state): + yield self._apply_transformations(record, self.config, stream_slice) def _apply_transformations(self, record: Mapping[str, Any], config: Config, stream_slice: StreamSlice): output_record = record - for transformation in self._transformations: + for transformation in self.transformations: output_record = transformation.transform(record, config=config, stream_state=self.state, stream_slice=stream_slice) return output_record @@ -116,7 +126,7 @@ def get_json_schema(self) -> Mapping[str, Any]: The default implementation of this method looks for a JSONSchema file with the same name as this stream's "name" property. Override as needed. """ - return self._schema_loader.get_json_schema() + return self.schema_loader.get_json_schema() def stream_slices( self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None @@ -130,4 +140,4 @@ def stream_slices( :return: """ # this is not passing the cursor field because it is known at init time - return self._retriever.stream_slices(sync_mode=sync_mode, stream_state=stream_state) + return self.retriever.stream_slices(sync_mode=sync_mode, stream_state=stream_state) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/__init__.py index 46b7376756ec..64a933247bdb 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/__init__.py @@ -1,3 +1,8 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder + +__all__ = ["Decoder", "JsonDecoder"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/decoder.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/decoder.py index 39a9b91c7747..5ec36516f4fd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/decoder.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/decoder.py @@ -3,11 +3,13 @@ # from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Any, List, Mapping, Union import requests +@dataclass class Decoder(ABC): """ Decoder strategy to transform a requests.Response into a Mapping[str, Any] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/json_decoder.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/json_decoder.py index 0e10d19805d8..0cea90365684 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/json_decoder.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/json_decoder.py @@ -2,16 +2,20 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, List, Mapping, Union import requests from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +@dataclass class JsonDecoder(Decoder): """ Decoder strategy that returns the json-encoded content of a response, if any. """ + options: InitVar[Mapping[str, Any]] + def decode(self, response: requests.Response) -> Union[Mapping[str, Any], List]: return response.json() or {} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py index 1100c1c58cf5..897f382ea0de 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py @@ -1,3 +1,10 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector +from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor +from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter +from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector + +__all__ = ["HttpSelector", "JelloExtractor", "RecordFilter", "RecordSelector"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py index dd02da0b42d9..517f61c70b79 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py @@ -3,12 +3,14 @@ # from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Any, List, Mapping, Optional import requests from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +@dataclass class HttpSelector(ABC): """ Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py index 250d712d26f6..f36613e2a56e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py @@ -2,40 +2,42 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import List, Union +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Union import requests from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Config, Record +from dataclasses_jsonschema import JsonSchemaMixin from jello import lib as jello_lib -class JelloExtractor: +@dataclass +class JelloExtractor(JsonSchemaMixin): """ Record extractor that evaluates a Jello query to extract records from a decoded response. More information on Jello can be found at https://github.com/kellyjonbrazil/jello + + Attributes: + transform (Union[InterpolatedString, str]): The Jello query to evaluate on the decoded response + config (Config): The user-provided configuration as specified by the source's spec + decoder (Decoder): The decoder responsible to transfom the response in a Mapping """ default_transform = "_" + transform: Union[InterpolatedString, str] + config: Config + options: InitVar[Mapping[str, Any]] + decoder: Decoder = JsonDecoder(options={}) - def __init__(self, transform: Union[InterpolatedString, str], config: Config, decoder: Decoder = JsonDecoder()): - """ - :param transform: The Jello query to evaluate on the decoded response - :param config: The user-provided configuration as specified by the source's spec - :param decoder: The decoder responsible to transfom the response in a Mapping - """ - - if isinstance(transform, str): - transform = InterpolatedString(transform, default=self.default_transform) - - self._transform = transform - self._decoder = decoder - self._config = config + def __post_init__(self, options: Mapping[str, Any]): + if isinstance(self.transform, str): + self.transform = InterpolatedString(string=self.transform, default=self.default_transform, options=options or {}) def extract_records(self, response: requests.Response) -> List[Record]: - response_body = self._decoder.decode(response) - script = self._transform.eval(self._config) + response_body = self.decoder.decode(response) + script = self.transform.eval(self.config) return jello_lib.pyquery(response_body, script) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py index 8f0b123ff895..081dd7597130 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py @@ -2,26 +2,29 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass, field from typing import Any, List, Mapping, Optional from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin -class RecordFilter: +@dataclass +class RecordFilter(JsonSchemaMixin): """ Filter applied on a list of Records + + config (Config): The user-provided configuration as specified by the source's spec + condition (str): The string representing the predicate to filter a record. Records will be removed if evaluated to False """ - def __init__(self, config: Config, condition: str = "", **options: Optional[Mapping[str, Any]]): - """ - :param config: The user-provided configuration as specified by the source's spec - :param condition: The string representing the predicate to filter a record. Records will be removed if evaluated to False - :param options: Additional runtime parameters to be used for string interpolation - """ - self._config = config - self._filter_interpolator = InterpolatedBoolean(condition) - self._options = options + options: InitVar[Mapping[str, Any]] + config: Config = field(default=dict) + condition: str = "" + + def __post_init__(self, options: Mapping[str, Any]): + self._filter_interpolator = InterpolatedBoolean(condition=self.condition, options=options) def filter_records( self, @@ -31,4 +34,4 @@ def filter_records( next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} - return [record for record in records if self._filter_interpolator.eval(self._config, record=record, **kwargs)] + return [record for record in records if self._filter_interpolator.eval(self.config, record=record, **kwargs)] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index 193f0e7576eb..8f27fb125b9c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, List, Mapping, Optional import requests @@ -9,22 +10,25 @@ from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin -class RecordSelector(HttpSelector): +@dataclass +class RecordSelector(HttpSelector, JsonSchemaMixin): """ Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering records based on a heuristic. + + Attributes: + extractor (JelloExtractor): The record extractor responsible for extracting records from a response + record_filter (RecordFilter): The record filter responsible for filtering extracted records """ - def __init__(self, extractor: JelloExtractor, record_filter: RecordFilter = None, **options: Optional[Mapping[str, Any]]): - """ - :param extractor: The record extractor responsible for extracting records from a response - :param record_filter: The record filter responsible for filtering extracted records - :param options: Additional runtime parameters to be used for string interpolation - """ - self._extractor = extractor - self._record_filter = record_filter + extractor: JelloExtractor + options: InitVar[Mapping[str, Any]] + record_filter: RecordFilter = None + + def __post_init__(self, options: Mapping[str, Any]): self._options = options def select_records( @@ -34,9 +38,9 @@ def select_records( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: - all_records = self._extractor.extract_records(response) - if self._record_filter: - return self._record_filter.filter_records( + all_records = self.extractor.extract_records(response) + if self.record_filter: + return self.record_filter.filter_records( all_records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) return all_records diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/__init__.py index 46b7376756ec..1f1b53a1910a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/__init__.py @@ -1,3 +1,9 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean +from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString + +__all__ = ["InterpolatedBoolean", "InterpolatedMapping", "InterpolatedString"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py index 8eff06e3bd8a..f7979dd69e46 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py @@ -2,26 +2,29 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, Final, List, Mapping, Optional +from dataclasses import InitVar, dataclass +from typing import Any, Final, List, Mapping from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation from airbyte_cdk.sources.declarative.types import Config +from dataclasses_jsonschema import JsonSchemaMixin FALSE_VALUES: Final[List[Any]] = ["False", "false", "{}", "[]", "()", "", "0", "0.0", "False", "false", {}, False, [], (), set()] -class InterpolatedBoolean: +@dataclass +class InterpolatedBoolean(JsonSchemaMixin): f""" Wrapper around a string to be evaluated to a boolean value. The string will be evaluated as False if it interpolates to a value in {FALSE_VALUES} + + Attributes: + condition (str): The string representing the condition to evaluate to a boolean """ + condition: str + options: InitVar[Mapping[str, Any]] - def __init__(self, condition: str, **options: Optional[Mapping[str, Any]]): - """ - :param condition: The string representing the condition to evaluate to a boolean - :param options: Additional runtime parameters to be used for string interpolation - """ - self._condition = condition + def __post_init__(self, options: Mapping[str, Any]): self._default = "False" self._interpolation = JinjaInterpolation() self._options = options @@ -34,10 +37,10 @@ def eval(self, config: Config, **additional_options): :param additional_options: Optional parameters used for interpolation :return: The interpolated string """ - if isinstance(self._condition, bool): - return self._condition + if isinstance(self.condition, bool): + return self.condition else: - evaluated = self._interpolation.eval(self._condition, config, self._default, options=self._options, **additional_options) + evaluated = self._interpolation.eval(self.condition, config, self._default, options=self._options, **additional_options) if evaluated in FALSE_VALUES: return False # The presence of a value is generally regarded as truthy, so we treat it as such diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py index fc46cf4f8552..6c1f80886a52 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py @@ -2,23 +2,30 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, Mapping + +from dataclasses import InitVar, dataclass +from typing import Any, Mapping, Optional from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation from airbyte_cdk.sources.declarative.types import Config +from dataclasses_jsonschema import JsonSchemaMixin -class InterpolatedMapping: - """Wrapper around a Mapping[str, str] where both the keys and values are to be interpolated.""" +@dataclass +class InterpolatedMapping(JsonSchemaMixin): + """ + Wrapper around a Mapping[str, str] where both the keys and values are to be interpolated. - def __init__(self, mapping: Mapping[str, Any], options: Mapping[str, Any]): - """ - :param mapping: Mapping[str, str] to be evaluated - :param options: Additional runtime parameters to be used for string interpolation - """ - self._mapping = mapping - self._options = options + Attributes: + mapping (Mapping[str, str]): to be evaluated + """ + + mapping: Mapping[str, str] + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Optional[Mapping[str, Any]]): self._interpolation = JinjaInterpolation() + self._options = options def eval(self, config: Config, **additional_options): """ @@ -32,7 +39,7 @@ def eval(self, config: Config, **additional_options): self._interpolation.eval(name, config, options=self._options, **additional_options): self._eval( value, config, **additional_options ) - for name, value in self._mapping.items() + for name, value in self.mapping.items() } return interpolated_values diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py index 3c2171a7d9d2..145be0d949d0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py @@ -2,27 +2,33 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, Mapping, Optional, Union from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation from airbyte_cdk.sources.declarative.types import Config +from dataclasses_jsonschema import JsonSchemaMixin -class InterpolatedString: +@dataclass +class InterpolatedString(JsonSchemaMixin): """ Wrapper around a raw string to be interpolated with the Jinja2 templating engine + + Attributes: + string (str): The string to evalute + default (Optional[str]): The default value to return if the evaluation returns an empty string + options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - def __init__(self, string: str, *, options: Mapping[str, Any] = {}, default: Optional[str] = None): - """ - :param string: The string to evalute - :param default: The default value to return if the evaluation returns an empty string - :param options: Additional runtime parameters to be used for string interpolation - """ - self._string = string - self._default = default or string + string: str + options: InitVar[Mapping[str, Any]] + default: Optional[str] = None + + def __post_init__(self, options: Mapping[str, Any]): + self.default = self.default or self.string self._interpolation = JinjaInterpolation() - self._options = options or {} + self._options = options def eval(self, config: Config, **kwargs): """ @@ -32,12 +38,12 @@ def eval(self, config: Config, **kwargs): :param kwargs: Optional parameters used for interpolation :return: The interpolated string """ - return self._interpolation.eval(self._string, config, self._default, options=self._options, **kwargs) + return self._interpolation.eval(self.string, config, self.default, options=self._options, **kwargs) def __eq__(self, other): if not isinstance(other, InterpolatedString): return False - return self._string == other._string and self._default == other._default + return self.string == other.string and self.default == other.default @classmethod def create( @@ -54,6 +60,6 @@ def create( :return: InterpolatedString representing the input string. """ if isinstance(string_or_interpolated, str): - return InterpolatedString(string_or_interpolated, options=options) + return InterpolatedString(string=string_or_interpolated, options=options) else: return string_or_interpolated diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/__init__.py index 46b7376756ec..ca8377e6fc97 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/__init__.py @@ -1,3 +1,9 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption +from airbyte_cdk.sources.declarative.requesters.requester import Requester + +__all__ = ["HttpRequester", "RequestOption", "Requester"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py index 1100c1c58cf5..f2602eea94b5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py @@ -1,3 +1,11 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy +from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import CompositeErrorHandler +from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler +from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler +from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter + +__all__ = ["BackoffStrategy", "CompositeErrorHandler", "DefaultErrorHandler", "ErrorHandler", "HttpResponseFilter"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py index 1100c1c58cf5..15472c2bd76a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py @@ -1,3 +1,21 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import ConstantBackoffStrategy +from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( + ExponentialBackoffStrategy, +) +from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_time_from_header_backoff_strategy import ( + WaitTimeFromHeaderBackoffStrategy, +) +from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_until_time_from_header_backoff_strategy import ( + WaitUntilTimeFromHeaderBackoffStrategy, +) + +__all__ = [ + "ConstantBackoffStrategy", + "ExponentialBackoffStrategy", + "WaitTimeFromHeaderBackoffStrategy", + "WaitUntilTimeFromHeaderBackoffStrategy", +] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py index d100dff0d2e4..3a7df2dc7b36 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py @@ -2,22 +2,24 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import dataclass from typing import Optional import requests from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy +from dataclasses_jsonschema import JsonSchemaMixin -class ConstantBackoffStrategy(BackoffStrategy): +@dataclass +class ConstantBackoffStrategy(BackoffStrategy, JsonSchemaMixin): """ Backoff strategy with a constant backoff interval + + Attributes: + backoff_time_in_seconds (float): time to backoff before retrying a retryable request. """ - def __init__(self, backoff_time_in_seconds: float): - """ - :param backoff_time_in_seconds: time to backoff before retrying a retryable request - """ - self._backoff_time_in_seconds = backoff_time_in_seconds + backoff_time_in_seconds: float def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]: - return self._backoff_time_in_seconds + return self.backoff_time_in_seconds diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py index 71c24ff3ea52..75a52ffca375 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py @@ -2,22 +2,24 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import dataclass from typing import Optional import requests from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy +from dataclasses_jsonschema import JsonSchemaMixin -class ExponentialBackoffStrategy(BackoffStrategy): +@dataclass +class ExponentialBackoffStrategy(BackoffStrategy, JsonSchemaMixin): """ Backoff strategy with an exponential backoff interval + + Attributes: + factor (float): multiplicative factor """ - def __init__(self, factor: float = 5): - """ - :param factor: multiplicative factor - """ - self._factor = factor + factor: float = 5 def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]: - return self._factor * 2**attempt_count + return self.factor * 2**attempt_count diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py index 7a30053554c6..3ff279c78eb5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py @@ -3,26 +3,31 @@ # import re +from dataclasses import dataclass from typing import Optional import requests from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import get_numeric_value_from_header from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy +from dataclasses_jsonschema import JsonSchemaMixin -class WaitTimeFromHeaderBackoffStrategy(BackoffStrategy): +@dataclass +class WaitTimeFromHeaderBackoffStrategy(BackoffStrategy, JsonSchemaMixin): """ Extract wait time from http header + + Attributes: + header (str): header to read wait time from + regex (Optional[str]): optional regex to apply on the header to extract its value """ - def __init__(self, header: str, regex: Optional[str] = None): - """ - :param header: header to read wait time from - :param regex: optional regex to apply on the header to extract its value - """ - self._header = header - self._regex = re.compile(regex) if regex else None + header: str + regex: Optional[str] = None + + def __post_init__(self): + self.regex = re.compile(self.regex) if self.regex else None def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]: - header_value = get_numeric_value_from_header(response, self._header, self._regex) + header_value = get_numeric_value_from_header(response, self.header, self.regex) return header_value diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py index a406bd5d0583..0e56741035ba 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py @@ -5,41 +5,45 @@ import numbers import re import time +from dataclasses import dataclass from typing import Optional import requests from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import get_numeric_value_from_header from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy +from dataclasses_jsonschema import JsonSchemaMixin -class WaitUntilTimeFromHeaderBackoffStrategy(BackoffStrategy): +@dataclass +class WaitUntilTimeFromHeaderBackoffStrategy(BackoffStrategy, JsonSchemaMixin): """ Extract time at which we can retry the request from response header and wait for the difference between now and that time + + Attributes: + header (str): header to read wait time from + min_wait (Optional[float]): minimum time to wait for safety + regex (Optional[str]): optional regex to apply on the header to extract its value """ - def __init__(self, header: str, min_wait: Optional[float] = None, regex: Optional[str] = None): - """ + header: str + min_wait: Optional[float] = None + regex: Optional[str] = None - :param header: header to read wait time from - :param min_wait: minimum time to wait for safety - :param regex: optional regex to apply on the header to extract its value - """ - self._header = header - self._min_wait = min_wait - self._regex = re.compile(regex) if regex else None + def __post_init__(self): + self.regex = re.compile(self.regex) if self.regex else None def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]: now = time.time() - wait_until = get_numeric_value_from_header(response, self._header, self._regex) + wait_until = get_numeric_value_from_header(response, self.header, self.regex) if wait_until is None or not wait_until: - return self._min_wait + return self.min_wait if (isinstance(wait_until, str) and wait_until.isnumeric()) or isinstance(wait_until, numbers.Number): wait_time = float(wait_until) - now else: - return self._min_wait - if self._min_wait: - return max(wait_time, self._min_wait) + return self.min_wait + if self.min_wait: + return max(wait_time, self.min_wait) elif wait_time < 0: return None return wait_time diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py index 55723e41f915..00c1b6dff23b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py @@ -3,11 +3,13 @@ # from abc import abstractmethod +from dataclasses import dataclass from typing import Optional import requests +@dataclass class BackoffStrategy: """ Backoff strategy defining how long to wait before retrying a request that resulted in an error. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index 906b16c3d81e..0c2cfe5da878 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -2,16 +2,19 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import List, Union +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Union import airbyte_cdk.sources.declarative.requesters.error_handlers.response_status as response_status import requests from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus +from dataclasses_jsonschema import JsonSchemaMixin -class CompositeErrorHandler(ErrorHandler): +@dataclass +class CompositeErrorHandler(ErrorHandler, JsonSchemaMixin): """ Error handler that sequentially iterates over a list of `ErrorHandler`s @@ -31,23 +34,24 @@ class CompositeErrorHandler(ErrorHandler): backoff_strategies: - type: "ConstantBackoffStrategy" backoff_time_in_seconds: 10 + Attributes: + error_handlers (List[ErrorHandler]): list of error handlers """ - def __init__(self, error_handlers: List[ErrorHandler]): - """ - :param error_handlers: list of error handlers - """ - self._error_handlers = error_handlers - if not self._error_handlers: + error_handlers: List[ErrorHandler] + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): + if not self.error_handlers: raise ValueError("CompositeErrorHandler expects at least 1 underlying error handler") @property def max_retries(self) -> Union[int, None]: - return self._error_handlers[0].max_retries + return self.error_handlers[0].max_retries def should_retry(self, response: requests.Response) -> ResponseStatus: should_retry = None - for retrier in self._error_handlers: + for retrier in self.error_handlers: should_retry = retrier.should_retry(response) if should_retry.action == ResponseAction.SUCCESS: return response_status.SUCCESS diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py index 5bcda8231c0f..179db638661f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py @@ -2,7 +2,8 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import List, MutableMapping, Optional, Union +from dataclasses import InitVar, dataclass, field +from typing import Any, List, Mapping, MutableMapping, Optional, Union import airbyte_cdk.sources.declarative.requesters.error_handlers.response_status as response_status import requests @@ -14,9 +15,11 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus +from dataclasses_jsonschema import JsonSchemaMixin -class DefaultErrorHandler(ErrorHandler): +@dataclass +class DefaultErrorHandler(ErrorHandler, JsonSchemaMixin): """ Default error handler. @@ -77,32 +80,33 @@ class DefaultErrorHandler(ErrorHandler): - http_codes: [ 404 ] action: RETRY ` + + Attributes: + response_filters (Optional[List[HttpResponseFilter]]): response filters to iterate on + max_retries (Optional[int]): maximum retry attempts + backoff_strategies (Optional[List[BackoffStrategy]]): list of backoff strategies to use to determine how long + to wait before retrying """ DEFAULT_BACKOFF_STRATEGY = ExponentialBackoffStrategy - def __init__( - self, - response_filters: Optional[List[HttpResponseFilter]] = None, - max_retries: Optional[int] = 5, - backoff_strategies: Optional[List[BackoffStrategy]] = None, - ): - """ - :param response_filters: response filters to iterate on - :param max_retries: maximum retry attemps - :param backoff_strategies: list of backoff strategies to use to determine how long to wait before retrying - """ - self._max_retries = max_retries - self._response_filters = response_filters or [] - - if not response_filters: - self._response_filters.append(HttpResponseFilter(ResponseAction.RETRY, http_codes=HttpResponseFilter.DEFAULT_RETRIABLE_ERRORS)) - self._response_filters.append(HttpResponseFilter(ResponseAction.IGNORE)) - - if backoff_strategies: - self._backoff_strategies = backoff_strategies - else: - self._backoff_strategies = [DefaultErrorHandler.DEFAULT_BACKOFF_STRATEGY()] + options: InitVar[Mapping[str, Any]] + response_filters: Optional[List[HttpResponseFilter]] = None + max_retries: Optional[int] = 5 + _max_retries: int = field(init=False, repr=False, default=5) + backoff_strategies: Optional[List[BackoffStrategy]] = None + + def __post_init__(self, options: Mapping[str, Any]): + self.response_filters = self.response_filters or [] + + if not self.response_filters: + self.response_filters.append( + HttpResponseFilter(ResponseAction.RETRY, http_codes=HttpResponseFilter.DEFAULT_RETRIABLE_ERRORS, options={}) + ) + self.response_filters.append(HttpResponseFilter(ResponseAction.IGNORE, options={})) + + if not self.backoff_strategies: + self.backoff_strategies = [DefaultErrorHandler.DEFAULT_BACKOFF_STRATEGY()] self._last_request_to_attempt_count: MutableMapping[requests.PreparedRequest, int] = {} @@ -110,6 +114,13 @@ def __init__( def max_retries(self) -> Union[int, None]: return self._max_retries + @max_retries.setter + def max_retries(self, value: Union[int, None]): + # Covers the case where max_retries is not provided in the constructor, which causes the property object + # to be set which we need to avoid doing + if not isinstance(value, property): + self._max_retries = value + def should_retry(self, response: requests.Response) -> ResponseStatus: request = response.request @@ -117,7 +128,7 @@ def should_retry(self, response: requests.Response) -> ResponseStatus: self._last_request_to_attempt_count = {request: 1} else: self._last_request_to_attempt_count[request] += 1 - for response_filter in self._response_filters: + for response_filter in self.response_filters: filter_action = response_filter.matches(response) if filter_action is not None: if filter_action == ResponseAction.RETRY: @@ -131,7 +142,7 @@ def should_retry(self, response: requests.Response) -> ResponseStatus: def _backoff_time(self, response: requests.Response, attempt_count: int) -> Optional[float]: backoff = None - for backoff_strategies in self._backoff_strategies: + for backoff_strategies in self.backoff_strategies: backoff = backoff_strategies.backoff(response, attempt_count) if backoff: return backoff diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py index f42e4ef9401a..50b6412ad350 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py @@ -3,12 +3,14 @@ # from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Union import requests from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus +@dataclass class ErrorHandler(ABC): """ Defines whether a request was successful and how to handle a failure. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py index 69790dad9f91..2da7f6272f21 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py @@ -2,42 +2,43 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Optional, Set, Union +from dataclasses import InitVar, dataclass +from typing import Any, Mapping, Optional, Set, Union import requests from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.streams.http.http import HttpStream +from dataclasses_jsonschema import JsonSchemaMixin -class HttpResponseFilter: +@dataclass +class HttpResponseFilter(JsonSchemaMixin): """ Filter to select HttpResponses + + Attributes: + action (Union[ResponseAction, str]): action to execute if a request matches + http_codes (Set[int]): http code of matching requests + error_message_contains (str): error substring of matching requests + predicate (str): predicate to apply to determine if a request is matching """ TOO_MANY_REQUESTS_ERRORS = {429} DEFAULT_RETRIABLE_ERRORS = set([x for x in range(500, 600)]).union(TOO_MANY_REQUESTS_ERRORS) - def __init__( - self, action: Union[ResponseAction, str], *, http_codes: Set[int] = None, error_message_contain: str = None, predicate: str = "" - ): - """ - :param action: action to execute if a request matches - :param http_codes: http code of matching requests - :param error_message_contain: error substring of matching requests - :param predicate: predicate to apply to determine if a request is matching - """ - if isinstance(action, str): - action = ResponseAction[action] - self._http_codes = http_codes or set() - self._predicate = InterpolatedBoolean(predicate) - self._error_message_contains = error_message_contain - self._action = action + action: Union[ResponseAction, str] + options: InitVar[Mapping[str, Any]] + http_codes: Set[int] = None + error_message_contains: str = None + predicate: Union[InterpolatedBoolean, str] = "" - @property - def action(self) -> ResponseAction: - """The ResponseAction to execute when a response matches the filter""" - return self._action + def __post_init__(self, options: Mapping[str, Any]): + if isinstance(self.action, str): + self.action = ResponseAction[self.action] + self.http_codes = self.http_codes or set() + if isinstance(self.predicate, str): + self.predicate = InterpolatedBoolean(condition=self.predicate, options=options) def matches(self, response: requests.Response) -> Optional[ResponseAction]: """ @@ -46,20 +47,20 @@ def matches(self, response: requests.Response) -> Optional[ResponseAction]: :return: The action to execute. None if the response does not match the filter """ if ( - response.status_code in self._http_codes + response.status_code in self.http_codes or (self._response_matches_predicate(response)) or (self._response_contains_error_message(response)) ): - return self._action + return self.action else: return None def _response_matches_predicate(self, response: requests.Response) -> bool: - return self._predicate and self._predicate.eval(None, response=response.json()) + return self.predicate and self.predicate.eval(None, response=response.json()) def _response_contains_error_message(self, response: requests.Response) -> bool: - if not self._error_message_contains: + if not self.error_message_contains: return False else: error_message = HttpStream.parse_response_error_message(response) - return error_message and self._error_message_contains in error_message + return error_message and self.error_message_contains in error_message diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py index 651b58186e5d..4658e66c704f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from functools import lru_cache from typing import Any, Mapping, MutableMapping, Optional, Union @@ -17,64 +18,66 @@ from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, NoAuth +from dataclasses_jsonschema import JsonSchemaMixin -class HttpRequester(Requester): +@dataclass +class HttpRequester(Requester, JsonSchemaMixin): """ Default implementation of a Requester + + Attributes: + name (str): Name of the stream. Only used for request/response caching + url_base (InterpolatedString): Base url to send requests to + path (InterpolatedString): Path to send requests to + http_method (Union[str, HttpMethod]): HTTP method to use when sending requests + request_options_provider (Optional[RequestOptionsProvider]): request option provider defining the options to set on outgoing requests + authenticator (HttpAuthenticator): Authenticator defining how to authenticate to the source + error_handler (Optional[ErrorHandler]): Error handler defining how to detect and handle errors + config (Config): The user-provided configuration as specified by the source's spec """ - def __init__( - self, - *, - name: str, - url_base: InterpolatedString, - path: InterpolatedString, - http_method: Union[str, HttpMethod] = HttpMethod.GET, - request_options_provider: Optional[RequestOptionsProvider] = None, - authenticator: HttpAuthenticator = None, - error_handler: Optional[ErrorHandler] = None, - config: Config, - **options: Optional[Mapping[str, Any]], - ): - """ - :param name: Name of the stream. Only used for request/response caching - :param url_base: Base url to send requests to - :param path: Path to send requests to - :param http_method: HTTP method to use when sending requests - :param request_options_provider: request option provider defining the options to set on outgoing requests - :param authenticator: Authenticator defining how to authenticate to the source - :param error_handler: Error handler defining how to detect and handle errors - :param config: The user-provided configuration as specified by the source's spec - :param options: Additional runtime parameters to be used for string interpolation - """ - if request_options_provider is None: - request_options_provider = InterpolatedRequestOptionsProvider(config=config) - elif isinstance(request_options_provider, dict): - request_options_provider = InterpolatedRequestOptionsProvider(config=config, **request_options_provider) - self._name = name - self._authenticator = authenticator or NoAuth() - self._url_base = url_base - self._path: InterpolatedString = path - if type(http_method) == str: - http_method = HttpMethod[http_method] - self._method = http_method - self._request_options_provider = request_options_provider - self._error_handler = error_handler or DefaultErrorHandler() - self._config = config + name: str + url_base: InterpolatedString + path: InterpolatedString + config: Config + options: InitVar[Mapping[str, Any]] + http_method: Union[str, HttpMethod] = HttpMethod.GET + request_options_provider: Optional[RequestOptionsProvider] = None + authenticator: HttpAuthenticator = None + error_handler: Optional[ErrorHandler] = None + + def __post_init__(self, options: Mapping[str, Any]): + if self.request_options_provider is None: + self._request_options_provider = InterpolatedRequestOptionsProvider(config=self.config, options=options) + elif isinstance(self.request_options_provider, dict): + self._request_options_provider = InterpolatedRequestOptionsProvider(config=self.config, **self.request_options_provider) + else: + self._request_options_provider = self.request_options_provider + self.authenticator = self.authenticator or NoAuth() + if type(self.http_method) == str: + self.http_method = HttpMethod[self.http_method] + self._method = self.http_method + self.error_handler = self.error_handler or DefaultErrorHandler(options=options) self._options = options + # We are using an LRU cache in should_retry() method which requires all incoming arguments (including self) to be hashable. + # Dataclasses by default are not hashable, so we need to define __hash__(). Alternatively, we can set @dataclass(frozen=True), + # but this has a cascading effect where all dataclass fields must also be set to frozen. + def __hash__(self): + return hash(tuple(self.__dict__)) + def get_authenticator(self): - return self._authenticator + return self.authenticator def get_url_base(self): - return self._url_base.eval(self._config) + return self.url_base.eval(self.config) def get_path( self, *, stream_state: Optional[StreamState], stream_slice: Optional[StreamSlice], next_page_token: Optional[Mapping[str, Any]] ) -> str: kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} - path = self._path.eval(self._config, **kwargs) + path = self.path.eval(self.config, **kwargs) return path def get_method(self): @@ -85,49 +88,49 @@ def get_method(self): @lru_cache(maxsize=10) def should_retry(self, response: requests.Response) -> ResponseStatus: # Cache the result because the HttpStream first checks if we should retry before looking at the backoff time - return self._error_handler.should_retry(response) + return self.error_handler.should_retry(response) - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - return self._request_options_provider.request_params( + return self._request_options_provider.get_request_params( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._request_options_provider.request_headers( + return self._request_options_provider.get_request_headers( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[Union[Mapping, str]]: - return self._request_options_provider.request_body_data( + return self._request_options_provider.get_request_body_data( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[Mapping]: - return self._request_options_provider.request_body_json( + return self._request_options_provider.get_request_body_json( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) @@ -145,7 +148,7 @@ def request_kwargs( @property def cache_filename(self) -> str: # FIXME: this should be declarative - return f"{self._name}.yml" + return f"{self.name}.yml" @property def use_cache(self) -> bool: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py index 46b7376756ec..d0310b21c199 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py @@ -1,3 +1,10 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.requesters.paginators.limit_paginator import LimitPaginator +from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination +from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy + +__all__ = ["LimitPaginator", "NoPagination", "PaginationStrategy", "Paginator"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py index 6d34cdd5a32d..675270f0da87 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py @@ -2,19 +2,22 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Optional +from dataclasses import InitVar, dataclass, field +from typing import Any, List, Mapping, Optional, Union import requests from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.paginators.pagination_strategy import PaginationStrategy from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin -class LimitPaginator(Paginator): +@dataclass +class LimitPaginator(Paginator, JsonSchemaMixin): """ Limit paginator to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token @@ -24,9 +27,9 @@ class LimitPaginator(Paginator): * updates the request path with "{{ response._metadata.next }}" paginator: type: "LimitPaginator" - limit_value: 10 + page_size: 10 limit_option: - option_type: request_parameter + inject_into: request_parameter field_name: page_size page_token_option: option_type: path @@ -41,9 +44,9 @@ class LimitPaginator(Paginator): ` paginator: type: "LimitPaginator" - limit_value: 5 + page_size: 5 limit_option: - option_type: header + inject_into: header field_name: page_size pagination_strategy: type: "OffsetIncrement" @@ -58,66 +61,57 @@ class LimitPaginator(Paginator): ` paginator: type: "LimitPaginator" - limit_value: 5 + page_size: 5 limit_option: - option_type: request_parameter + inject_into: request_parameter field_name: page_size pagination_strategy: type: "PageIncrement" page_token: option_type: "request_parameter" field_name: "page" + + Attributes: + page_size (int): the number of records to request + limit_option (RequestOption): the request option to set the limit. Cannot be injected in the path. + page_token_option (RequestOption): the request option to set the page token + pagination_strategy (PaginationStrategy): Strategy defining how to get the next page token + config (Config): connection config + url_base (Union[InterpolatedString, str]): endpoint's base url + decoder (Decoder): decoder to decode the response """ - def __init__( - self, - page_size: int, - limit_option: RequestOption, - page_token_option: RequestOption, - pagination_strategy: PaginationStrategy, - config: Config, - url_base: str, - decoder: Decoder = None, - **options: Optional[Mapping[str, Any]], - ): - """ - :param page_size: The number of records to request - :param limit_option: The request option to set the limit. Cannot be injected in the path. - :param page_token_option: The request option to set the page token - :param pagination_strategy: The strategy defining how to get the next page token - :param config: The user-provided configuration as specified by the source's spec - :param url_base: The endpoint's base url - :param decoder: The decoder to decode the response - :param options: Additional runtime parameters to be used for string interpolation - """ - if limit_option.inject_into == RequestOptionType.path: + page_size: int + limit_option: RequestOption + page_token_option: RequestOption + pagination_strategy: PaginationStrategy + config: Config + url_base: Union[InterpolatedString, str] + options: InitVar[Mapping[str, Any]] + decoder: Decoder = JsonDecoder(options={}) + _token: Optional[Any] = field(init=False, repr=False, default=None) + + def __post_init__(self, options: Mapping[str, Any]): + if self.limit_option.inject_into == RequestOptionType.path: raise ValueError("Limit parameter cannot be a path") - self._page_size = page_size - self._config = config - self._limit_option = limit_option - self._page_token_option = page_token_option - self._pagination_strategy = pagination_strategy - self._token = None - if isinstance(url_base, str): - url_base = InterpolatedString.create(url_base, options=options) - self._url_base = url_base - self._decoder = decoder or JsonDecoder() + if isinstance(self.url_base, str): + self.url_base = InterpolatedString(string=self.url_base, options=options) def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: - self._token = self._pagination_strategy.next_page_token(response, last_records) + self._token = self.pagination_strategy.next_page_token(response, last_records) if self._token: return {"next_page_token": self._token} else: return None def path(self): - if self._token and self._page_token_option.inject_into == RequestOptionType.path: + if self._token and self.page_token_option.inject_into == RequestOptionType.path: # Replace url base to only return the path - return str(self._token).replace(self._url_base.eval(self._config), "") + return str(self._token).replace(self.url_base.eval(self.config), "") else: return None - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, @@ -126,7 +120,7 @@ def request_params( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.request_parameter) - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, @@ -135,7 +129,7 @@ def request_headers( ) -> Mapping[str, str]: return self._get_request_options(RequestOptionType.header) - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, @@ -144,7 +138,7 @@ def request_body_data( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.body_data) - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, @@ -155,10 +149,10 @@ def request_body_json( def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]: options = {} - if self._page_token_option.inject_into == option_type: + if self.page_token_option.inject_into == option_type: if option_type != RequestOptionType.path and self._token: - options[self._page_token_option.field_name] = self._token - if self._limit_option.inject_into == option_type: + options[self.page_token_option.field_name] = self._token + if self.limit_option.inject_into == option_type: if option_type != RequestOptionType.path: - options[self._limit_option.field_name] = self._page_size + options[self.limit_option.field_name] = self.page_size return options diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py index 8877c829a7a2..ac54ba0bc70e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, List, Mapping, Optional, Union import requests @@ -9,15 +10,18 @@ from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState +@dataclass class NoPagination(Paginator): """ Pagination implementation that never returns a next page. """ + options: InitVar[Mapping[str, Any]] + def path(self) -> Optional[str]: return None - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, @@ -26,7 +30,7 @@ def request_params( ) -> Mapping[str, Any]: return {} - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, @@ -35,7 +39,7 @@ def request_headers( ) -> Mapping[str, str]: return {} - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, @@ -44,7 +48,7 @@ def request_body_data( ) -> Union[Mapping[str, Any], str]: return {} - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py index 084d3b6a5e88..e77ca744b3ed 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py @@ -3,12 +3,14 @@ # from abc import abstractmethod +from dataclasses import dataclass from typing import Any, List, Mapping, Optional import requests from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider +@dataclass class Paginator(RequestOptionsProvider): """ Defines the token to use to fetch the next page of records from the API. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py index 1100c1c58cf5..4b4f9d259d9b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py @@ -1,3 +1,9 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement + +__all__ = ["CursorPaginationStrategy", "OffsetIncrement", "PageIncrement"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index f583a28d6011..09d036580f8f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, List, Mapping, Optional, Union import requests @@ -9,43 +10,39 @@ from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.paginators.pagination_strategy import PaginationStrategy +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy from airbyte_cdk.sources.declarative.types import Config +from dataclasses_jsonschema import JsonSchemaMixin -class CursorPaginationStrategy(PaginationStrategy): +@dataclass +class CursorPaginationStrategy(PaginationStrategy, JsonSchemaMixin): """ Pagination strategy that evaluates an interpolated string to define the next page token + + Attributes: + cursor_value (Union[InterpolatedString, str]): template string evaluating to the cursor value + config (Config): connection config + stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating + decoder (Decoder): decoder to decode the response """ - def __init__( - self, - cursor_value: Union[InterpolatedString, str], - config: Config, - stop_condition: Optional[InterpolatedBoolean] = None, - decoder: Optional[Decoder] = None, - **options: Optional[Mapping[str, Any]], - ): - """ - :param cursor_value: template string evaluating to the cursor value - :param config: connection config - :param stop_condition: template string evaluating when to stop paginating - :param decoder: decoder to decode the response - :param options: Additional runtime parameters to be used for string interpolation - """ - if isinstance(cursor_value, str): - cursor_value = InterpolatedString.create(cursor_value, options=options) - self._cursor_value = cursor_value - self._config = config - self._decoder = decoder or JsonDecoder() - self._stop_condition = stop_condition + cursor_value: Union[InterpolatedString, str] + config: Config + options: InitVar[Mapping[str, Any]] + stop_condition: Optional[InterpolatedBoolean] = None + decoder: Decoder = JsonDecoder(options={}) + + def __post_init__(self, options: Mapping[str, Any]): + if isinstance(self.cursor_value, str): + self.cursor_value = InterpolatedString.create(self.cursor_value, options=options) def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: - decoded_response = self._decoder.decode(response) + decoded_response = self.decoder.decode(response) headers = response.headers - if self._stop_condition: - should_stop = self._stop_condition.eval(self._config, response=decoded_response, headers=headers, last_records=last_records) + if self.stop_condition: + should_stop = self.stop_condition.eval(self.config, response=decoded_response, headers=headers, last_records=last_records) if should_stop: return None - token = self._cursor_value.eval(config=self._config, last_records=last_records, response=decoded_response) + token = self.cursor_value.eval(config=self.config, last_records=last_records, response=decoded_response) return token if token else None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py index ae5cd6cff014..bfbd92df3e24 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py @@ -2,26 +2,31 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, List, Mapping, Optional import requests -from airbyte_cdk.sources.declarative.requesters.paginators.pagination_strategy import PaginationStrategy +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy +from dataclasses_jsonschema import JsonSchemaMixin -class OffsetIncrement(PaginationStrategy): +@dataclass +class OffsetIncrement(PaginationStrategy, JsonSchemaMixin): """ Pagination strategy that returns the number of records reads so far and returns it as the next page token + + Attributes: + page_size (int): the number of records to request """ - def __init__(self, page_size: int): - """ - :param page_size: the number of records to request - """ + page_size: int + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): self._offset = 0 - self._page_size = page_size def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: - if len(last_records) < self._page_size: + if len(last_records) < self.page_size: return None else: self._offset += len(last_records) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index e53479444cb5..f39ca388ada1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -2,26 +2,31 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, List, Mapping, Optional import requests -from airbyte_cdk.sources.declarative.requesters.paginators.pagination_strategy import PaginationStrategy +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy +from dataclasses_jsonschema import JsonSchemaMixin -class PageIncrement(PaginationStrategy): +@dataclass +class PageIncrement(PaginationStrategy, JsonSchemaMixin): """ Pagination strategy that returns the number of pages reads so far and returns it as the next page token + + Attributes: + page_size (int): the number of records to request """ - def __init__(self, page_size: int): - """ - :param page_size: the number of records to request - """ - self._page_size = page_size + page_size: int + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): self._offset = 0 def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: - if len(last_records) < self._page_size: + if len(last_records) < self.page_size: return None else: self._offset += 1 diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py similarity index 80% rename from airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/pagination_strategy.py rename to airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py index d839f5c35970..7174fc16a377 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py @@ -3,12 +3,15 @@ # from abc import abstractmethod +from dataclasses import dataclass from typing import Any, List, Mapping, Optional import requests +from dataclasses_jsonschema import JsonSchemaMixin -class PaginationStrategy: +@dataclass +class PaginationStrategy(JsonSchemaMixin): """ Defines how to get the next page token """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py index 91221557cf4a..1ed01f34b87c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_option.py @@ -2,8 +2,11 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from enum import Enum -from typing import Optional +from typing import Any, Mapping, Optional + +from dataclasses_jsonschema import JsonSchemaMixin class RequestOptionType(Enum): @@ -18,34 +21,27 @@ class RequestOptionType(Enum): body_json = "body_json" -class RequestOption: +@dataclass +class RequestOption(JsonSchemaMixin): """ Describes an option to set on a request + + Attributes: + inject_into (RequestOptionType): Describes where in the HTTP request to inject the parameter + field_name (Optional[str]): Describes the name of the parameter to inject. None if option_type == path. Required otherwise. """ - def __init__(self, inject_into: RequestOptionType, field_name: Optional[str] = None): - """ - :param inject_into: where to set the value - :param field_name: field name to set. None if option_type == path. Required otherwise. - """ - self._option_type = inject_into - self._field_name = field_name - if self._option_type == RequestOptionType.path: - if self._field_name is not None: - raise ValueError(f"RequestOption with path cannot have a field name. Get {field_name}") - elif self._field_name is None: - raise ValueError(f"RequestOption expected field name for type {self._option_type}") - - @property - def inject_into(self) -> RequestOptionType: - """Describes where in the HTTP request to inject the parameter""" - return self._option_type - - @property - def field_name(self) -> Optional[str]: - """Describes the name of the parameter to inject""" - return self._field_name + inject_into: RequestOptionType + options: InitVar[Mapping[str, Any]] + field_name: Optional[str] = None + + def __post_init__(self, options: Mapping[str, Any]): + if self.inject_into == RequestOptionType.path: + if self.field_name is not None: + raise ValueError(f"RequestOption with path cannot have a field name. Get {self.field_name}") + elif self.field_name is None: + raise ValueError(f"RequestOption expected field name for type {self.inject_into}") def is_path(self) -> bool: """Returns true if the parameter is the path to send the request to""" - return self._option_type == RequestOptionType.path + return self.inject_into == RequestOptionType.path diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py index 1100c1c58cf5..9bb93d757d12 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py @@ -1,3 +1,10 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import ( + InterpolatedRequestOptionsProvider, +) +from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider + +__all__ = ["InterpolatedRequestOptionsProvider", "RequestOptionsProvider"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/interpolated_request_input_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py similarity index 100% rename from airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/interpolated_request_input_provider.py rename to airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py index 793594eb4c11..6348ccc35884 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py @@ -2,52 +2,56 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass, field from typing import Any, Mapping, MutableMapping, Optional, Union -from airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider import InterpolatedRequestInputProvider +from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin RequestInput = Union[str, Mapping[str, str]] -class InterpolatedRequestOptionsProvider(RequestOptionsProvider): - """Defines the request options to set on an outgoing HTTP request by evaluating `InterpolatedMapping`s""" +@dataclass +class InterpolatedRequestOptionsProvider(RequestOptionsProvider, JsonSchemaMixin): + """ + Defines the request options to set on an outgoing HTTP request by evaluating `InterpolatedMapping`s - def __init__( - self, - *, - config: Config, - request_parameters: Optional[RequestInput] = None, - request_headers: Optional[RequestInput] = None, - request_body_data: Optional[RequestInput] = None, - request_body_json: Optional[RequestInput] = None, - ): - """ - :param config: The user-provided configuration as specified by the source's spec - :param request_parameters: The request parameters to set on an outgoing HTTP request - :param request_headers: The request headers to set on an outgoing HTTP request - :param request_body_data: The body data to set on an outgoing HTTP request - :param request_body_json: The json content to set on an outgoing HTTP request - """ - if request_parameters is None: - request_parameters = {} - if request_headers is None: - request_headers = {} - if request_body_data is None: - request_body_data = {} - if request_body_json is None: - request_body_json = {} + Attributes: + config (Config): The user-provided configuration as specified by the source's spec + request_parameters (Union[str, Mapping[str, str]]): The request parameters to set on an outgoing HTTP request + request_headers (Union[str, Mapping[str, str]]): The request headers to set on an outgoing HTTP request + request_body_data (Union[str, Mapping[str, str]]): The body data to set on an outgoing HTTP request + request_body_json (Union[str, Mapping[str, str]]): The json content to set on an outgoing HTTP request + """ + + options: InitVar[Mapping[str, Any]] + config: Config = field(default_factory=dict) + request_parameters: Optional[RequestInput] = None + request_headers: Optional[RequestInput] = None + request_body_data: Optional[RequestInput] = None + request_body_json: Optional[RequestInput] = None + + def __post_init__(self, options: Mapping[str, Any]): + if self.request_parameters is None: + self.request_parameters = {} + if self.request_headers is None: + self.request_headers = {} + if self.request_body_data is None: + self.request_body_data = {} + if self.request_body_json is None: + self.request_body_json = {} - if request_body_json and request_body_data: + if self.request_body_json and self.request_body_data: raise ValueError("RequestOptionsProvider should only contain either 'request_body_data' or 'request_body_json' not both") - self._parameter_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_parameters) - self._headers_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_headers) - self._body_data_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_data) - self._body_json_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_json) + self._parameter_interpolator = InterpolatedRequestInputProvider(config=self.config, request_inputs=self.request_parameters) + self._headers_interpolator = InterpolatedRequestInputProvider(config=self.config, request_inputs=self.request_headers) + self._body_data_interpolator = InterpolatedRequestInputProvider(config=self.config, request_inputs=self.request_body_data) + self._body_json_interpolator = InterpolatedRequestInputProvider(config=self.config, request_inputs=self.request_body_json) - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, @@ -59,7 +63,7 @@ def request_params( return interpolated_value return {} - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, @@ -68,7 +72,7 @@ def request_headers( ) -> Mapping[str, Any]: return self._headers_interpolator.request_inputs(stream_state, stream_slice, next_page_token) - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, @@ -77,7 +81,7 @@ def request_body_data( ) -> Optional[Union[Mapping, str]]: return self._body_data_interpolator.request_inputs(stream_state, stream_slice, next_page_token) - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py index 425107afe29a..1be5fa690349 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py @@ -3,11 +3,13 @@ # from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Any, Mapping, MutableMapping, Optional, Union from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState +@dataclass class RequestOptionsProvider(ABC): """ Defines the request options to set on an outgoing HTTP request @@ -20,7 +22,7 @@ class RequestOptionsProvider(ABC): """ @abstractmethod - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, @@ -35,7 +37,7 @@ def request_params( pass @abstractmethod - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, @@ -45,7 +47,7 @@ def request_headers( """Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.""" @abstractmethod - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, @@ -63,7 +65,7 @@ def request_body_data( """ @abstractmethod - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py index 8b7d0e045043..24c4211df5ed 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py @@ -55,7 +55,7 @@ def get_method(self) -> HttpMethod: """ @abstractmethod - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, @@ -81,7 +81,7 @@ def should_retry(self, response: requests.Response) -> ResponseStatus: """ @abstractmethod - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, @@ -93,7 +93,7 @@ def request_headers( """ @abstractmethod - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, @@ -111,7 +111,7 @@ def request_body_data( """ @abstractmethod - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/__init__.py index 1100c1c58cf5..9c47818b3e72 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/__init__.py @@ -1,3 +1,8 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever +from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever + +__all__ = ["Retriever", "SimpleRetriever"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py index bda876c52951..a9ae02806425 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py @@ -3,12 +3,14 @@ # from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Iterable, List, Optional from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +@dataclass class Retriever(ABC): """ Responsible for fetching a stream's records from an HTTP API source. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index b39a01d14a72..8eda1ec15401 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass, field from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import requests @@ -17,9 +18,11 @@ from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState from airbyte_cdk.sources.streams.http import HttpStream +from dataclasses_jsonschema import JsonSchemaMixin -class SimpleRetriever(Retriever, HttpStream): +@dataclass +class SimpleRetriever(Retriever, HttpStream, JsonSchemaMixin): """ Retrieves records by synchronously sending requests to fetch records. @@ -30,34 +33,30 @@ class SimpleRetriever(Retriever, HttpStream): This retriever currently inherits from HttpStream to reuse the request submission and pagination machinery. As a result, some of the parameters passed to some methods are unused. The two will be decoupled in a future release. + + Attributes: + stream_name (str): The stream's name + stream_primary_key (Optional[Union[str, List[str], List[List[str]]]]): The stream's primary key + requester (Requester): The HTTP requester + record_selector (HttpSelector): The record selector + paginator (Optional[Paginator]): The paginator + stream_slicer (Optional[StreamSlicer]): The stream slicer + options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - def __init__( - self, - name, - primary_key, - requester: Requester, - record_selector: HttpSelector, - paginator: Optional[Paginator] = None, - stream_slicer: Optional[StreamSlicer] = SingleSlice(), - **options: Optional[Mapping[str, Any]], - ): - """ - :param name: The stream's name - :param primary_key: The stream's primary key - :param requester: The HTTP requester - :param record_selector: The record selector - :param paginator: The paginator - :param stream_slicer: The stream slicer - :param options: Additional runtime parameters to be used for string interpolation - """ - self._name = name - self._primary_key = primary_key - self._paginator = paginator or NoPagination() - self._requester = requester - self._record_selector = record_selector - super().__init__(self._requester.get_authenticator()) - self._stream_slicer = stream_slicer + requester: Requester + record_selector: HttpSelector + options: InitVar[Mapping[str, Any]] + name: str + _name: str = field(init=False, repr=False) + primary_key: Optional[Union[str, List[str], List[List[str]]]] + _primary_key: str = field(init=False, repr=False) + paginator: Optional[Paginator] = None + stream_slicer: Optional[StreamSlicer] = SingleSlice(options={}) + + def __post_init__(self, options: Mapping[str, Any]): + self.paginator = self.paginator or NoPagination(options=options) + HttpStream.__init__(self, self.requester.get_authenticator()) self._last_response = None self._last_records = None @@ -68,13 +67,18 @@ def name(self) -> str: """ return self._name + @name.setter + def name(self, value: str) -> None: + if not isinstance(value, property): + self._name = value + @property def url_base(self) -> str: - return self._requester.get_url_base() + return self.requester.get_url_base() @property def http_method(self) -> str: - return str(self._requester.get_method().value) + return str(self.requester.get_method().value) @property def raise_on_http_errors(self) -> bool: @@ -91,7 +95,7 @@ def should_retry(self, response: requests.Response) -> bool: Unexpected but transient exceptions (connection timeout, DNS resolution failed, etc..) are retried by default. """ - return self._requester.should_retry(response).action == ResponseAction.RETRY + return self.requester.should_retry(response).action == ResponseAction.RETRY def backoff_time(self, response: requests.Response) -> Optional[float]: """ @@ -103,7 +107,7 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: :return how long to backoff in seconds. The return value may be a floating point number for subsecond precision. Returning None defers backoff to the default backoff behavior (e.g using an exponential algorithm). """ - should_retry = self._requester.should_retry(response) + should_retry = self.requester.should_retry(response) if should_retry.action != ResponseAction.RETRY: raise ValueError(f"backoff_time can only be applied on retriable response action. Got {should_retry.action}") assert should_retry.action == ResponseAction.RETRY @@ -127,11 +131,12 @@ def _get_request_options( :param paginator_method: :return: """ - requester_mapping = requester_method(self.state, stream_slice, next_page_token) + + requester_mapping = requester_method(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) requester_mapping_keys = set(requester_mapping.keys()) - paginator_mapping = paginator_method(self.state, stream_slice, next_page_token) + paginator_mapping = paginator_method(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) paginator_mapping_keys = set(paginator_mapping.keys()) - stream_slicer_mapping = stream_slicer_method(stream_slice) + stream_slicer_mapping = stream_slicer_method(stream_slice=stream_slice) stream_slicer_mapping_keys = set(stream_slicer_mapping.keys()) intersection = ( @@ -153,9 +158,9 @@ def request_headers( return self._get_request_options( stream_slice, next_page_token, - self._requester.request_headers, - self._paginator.request_headers, - self._stream_slicer.request_headers, + self.requester.get_request_headers, + self.paginator.get_request_headers, + self.stream_slicer.get_request_headers, ) def request_params( @@ -172,9 +177,9 @@ def request_params( return self._get_request_options( stream_slice, next_page_token, - self._requester.request_params, - self._paginator.request_params, - self._stream_slicer.request_params, + self.requester.get_request_params, + self.paginator.get_request_params, + self.stream_slicer.get_request_params, ) def request_body_data( @@ -193,11 +198,11 @@ def request_body_data( At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden. """ # Warning: use self.state instead of the stream_state passed as argument! - base_body_data = self._requester.request_body_data( + base_body_data = self.requester.get_request_body_data( stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token ) if isinstance(base_body_data, str): - paginator_body_data = self._paginator.request_body_data() + paginator_body_data = self.paginator.get_request_body_data() if paginator_body_data: raise ValueError( f"Cannot combine requester's body data= {base_body_data} with paginator's body_data: {paginator_body_data}" @@ -207,9 +212,9 @@ def request_body_data( return self._get_request_options( stream_slice, next_page_token, - self._requester.request_body_data, - self._paginator.request_body_data, - self._stream_slicer.request_body_data, + self.requester.get_request_body_data, + self.paginator.get_request_body_data, + self.stream_slicer.get_request_body_data, ) def request_body_json( @@ -227,9 +232,9 @@ def request_body_json( return self._get_request_options( stream_slice, next_page_token, - self._requester.request_body_json, - self._paginator.request_body_json, - self._stream_slicer.request_body_json, + self.requester.get_request_body_json, + self.paginator.get_request_body_json, + self.stream_slicer.get_request_body_json, ) def request_kwargs( @@ -244,7 +249,7 @@ def request_kwargs( this method. Note that these options do not conflict with request-level options such as headers, request params, etc.. """ # Warning: use self.state instead of the stream_state passed as argument! - return self._requester.request_kwargs(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) + return self.requester.request_kwargs(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) def path( self, @@ -262,25 +267,25 @@ def path( :return: """ # Warning: use self.state instead of the stream_state passed as argument! - paginator_path = self._paginator.path() + paginator_path = self.paginator.path() if paginator_path: return paginator_path else: - return self._requester.get_path(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) + return self.requester.get_path(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) @property def cache_filename(self) -> str: """ Return the name of cache file """ - return self._requester.cache_filename + return self.requester.cache_filename @property def use_cache(self) -> bool: """ If True, all records will be cached. """ - return self._requester.use_cache + return self.requester.use_cache def parse_response( self, @@ -293,7 +298,7 @@ def parse_response( # if fail -> raise exception # if ignore -> ignore response and return no records # else -> delegate to record selector - response_status = self._requester.should_retry(response) + response_status = self.requester.should_retry(response) if response_status.action == ResponseAction.FAIL: raise ReadException(f"Request {response.request} failed with response {response}") elif response_status.action == ResponseAction.IGNORE: @@ -302,7 +307,7 @@ def parse_response( # Warning: use self.state instead of the stream_state passed as argument! self._last_response = response - records = self._record_selector.select_records( + records = self.record_selector.select_records( response=response, stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token ) self._last_records = records @@ -313,6 +318,11 @@ def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: """The stream's primary key""" return self._primary_key + @primary_key.setter + def primary_key(self, value: str) -> None: + if not isinstance(value, property): + self._primary_key = value + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ Specifies a pagination strategy. @@ -321,7 +331,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, :return: The token for the next page from the input response object. Returning None means there are no more pages to read in this response. """ - return self._paginator.next_page_token(response, self._last_records) + return self.paginator.next_page_token(response, self._last_records) def read_records( self, @@ -334,11 +344,11 @@ def read_records( stream_slice = stream_slice or {} # None-check records_generator = HttpStream.read_records(self, sync_mode, cursor_field, stream_slice, self.state) for r in records_generator: - self._stream_slicer.update_cursor(stream_slice, last_record=r) + self.stream_slicer.update_cursor(stream_slice, last_record=r) yield r else: last_record = self._last_records[-1] if self._last_records else None - self._stream_slicer.update_cursor(stream_slice, last_record=last_record) + self.stream_slicer.update_cursor(stream_slice, last_record=last_record) yield from [] def stream_slices( @@ -353,13 +363,13 @@ def stream_slices( :return: """ # Warning: use self.state instead of the stream_state passed as argument! - return self._stream_slicer.stream_slices(sync_mode, self.state) + return self.stream_slicer.stream_slices(sync_mode, self.state) @property def state(self) -> MutableMapping[str, Any]: - return self._stream_slicer.get_stream_state() + return self.stream_slicer.get_stream_state() @state.setter def state(self, value: StreamState): """State setter, accept state serialized by state getter.""" - self._stream_slicer.update_cursor(value) + self.stream_slicer.update_cursor(value) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/__init__.py index 1100c1c58cf5..cbef6eb1d268 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/__init__.py @@ -1,3 +1,8 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.schema.json_schema import JsonSchema +from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader + +__all__ = ["JsonSchema", "SchemaLoader"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py index 8e29bd329c71..e3a42dd04f17 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py @@ -3,25 +3,34 @@ # import json -from typing import Any, Mapping, Optional +from dataclasses import InitVar, dataclass +from typing import Any, Mapping, Union from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader from airbyte_cdk.sources.declarative.types import Config +from dataclasses_jsonschema import JsonSchemaMixin -class JsonSchema(SchemaLoader): - """Loads the schema from a json file""" +@dataclass +class JsonSchema(SchemaLoader, JsonSchemaMixin): + """ + Loads the schema from a json file - def __init__(self, file_path: InterpolatedString, config: Config, **options: Optional[Mapping[str, Any]]): - """ - :param file_path: The path to the json file describing the schema - :param config: The user-provided configuration as specified by the source's spec - :param options: Additional arguments to pass to the string interpolation if needed - """ - self._file_path = file_path - self._config = config - self._options = options + Attributes: + file_path (Union[InterpolatedString, str]): The path to the json file describing the schema + name (str): The stream's name + config (Config): The user-provided configuration as specified by the source's spec + options (Mapping[str, Any]): Additional arguments to pass to the string interpolation if needed + """ + + file_path: Union[InterpolatedString, str] + name: str + config: Config + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): + self.file_path = InterpolatedString.create(self.file_path, options=options) def get_json_schema(self) -> Mapping[str, Any]: json_schema_path = self._get_json_filepath() @@ -29,4 +38,4 @@ def get_json_schema(self) -> Mapping[str, Any]: return json.loads(f.read()) def _get_json_filepath(self): - return self._file_path.eval(self._config, **self._options) + return self.file_path.eval(self.config) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/schema_loader.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/schema_loader.py index 57ce7ca8b0b7..3a0d45316a4e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/schema_loader.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/schema_loader.py @@ -3,9 +3,11 @@ # from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Any, Mapping +@dataclass class SchemaLoader(ABC): """Describes a stream's schema""" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/__init__.py index 1100c1c58cf5..5fcd546f87bd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/__init__.py @@ -1,3 +1,12 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.sources.declarative.stream_slicers.cartesian_product_stream_slicer import CartesianProductStreamSlicer +from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer +from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer +from airbyte_cdk.sources.declarative.stream_slicers.single_slice import SingleSlice +from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer +from airbyte_cdk.sources.declarative.stream_slicers.substream_slicer import SubstreamSlicer + +__all__ = ["CartesianProductStreamSlicer", "DatetimeStreamSlicer", "ListStreamSlicer", "SingleSlice", "StreamSlicer", "SubstreamSlicer"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py index 9c52c07abc87..1004b6a7ecee 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py @@ -4,14 +4,17 @@ import itertools from collections import ChainMap +from dataclasses import InitVar, dataclass from typing import Any, Iterable, List, Mapping, Optional from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin -class CartesianProductStreamSlicer(StreamSlicer): +@dataclass +class CartesianProductStreamSlicer(StreamSlicer, JsonSchemaMixin): """ Stream slicers that iterates over the cartesian product of input stream slicers Given 2 stream slicers with the following slices: @@ -26,57 +29,78 @@ class CartesianProductStreamSlicer(StreamSlicer): {"i": 2, "s": "hello"}, {"i": 2, "s": "world"}, ] + + Attributes: + stream_slicers (List[StreamSlicer]): Underlying stream slicers. The RequestOptions (e.g: Request headers, parameters, etc..) returned by this slicer are the combination of the RequestOptions of its input slicers. If there are conflicts e.g: two slicers define the same header or request param, the conflict is resolved by taking the value from the first slicer, where ordering is determined by the order in which slicers were input to this composite slicer. """ - def __init__(self, stream_slicers: List[StreamSlicer]): - """ - :param stream_slicers: Underlying stream slicers. The RequestOptions (e.g: Request headers, parameters, etc..) returned by this slicer are the combination of the RequestOptions of its input slicers. If there are conflicts e.g: two slicers define the same header or request param, the conflict is resolved by taking the value from the first slicer, where ordering is determined by the order in which slicers were input to this composite slicer. - """ - self._stream_slicers = stream_slicers + stream_slicers: List[StreamSlicer] + options: InitVar[Mapping[str, Any]] def update_cursor(self, stream_slice: Mapping[str, Any], last_record: Optional[Mapping[str, Any]] = None): - for slicer in self._stream_slicers: + for slicer in self.stream_slicers: slicer.update_cursor(stream_slice, last_record) - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return dict(ChainMap(*[s.request_params() for s in self._stream_slicers])) + return dict(ChainMap(*[s.get_request_params() for s in self.stream_slicers])) - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return dict(ChainMap(*[s.request_headers(stream_state, stream_slice, next_page_token) for s in self._stream_slicers])) + return dict( + ChainMap( + *[ + s.get_request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + for s in self.stream_slicers + ] + ) + ) - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return dict(ChainMap(*[s.request_body_data(stream_state, stream_slice, next_page_token) for s in self._stream_slicers])) + return dict( + ChainMap( + *[ + s.get_request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + for s in self.stream_slicers + ] + ) + ) - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[Mapping]: - return dict(ChainMap(*[s.request_body_json(stream_state, stream_slice, next_page_token) for s in self._stream_slicers])) + return dict( + ChainMap( + *[ + s.get_request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + for s in self.stream_slicers + ] + ) + ) def get_stream_state(self) -> Mapping[str, Any]: - return dict(ChainMap(*[slicer.get_stream_state() for slicer in self._stream_slicers])) + return dict(ChainMap(*[slicer.get_stream_state() for slicer in self.stream_slicers])) def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: - sub_slices = (s.stream_slices(sync_mode, stream_state) for s in self._stream_slicers) + sub_slices = (s.stream_slices(sync_mode, stream_state) for s in self.stream_slicers) return (ChainMap(*a) for a in itertools.product(*sub_slices)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py index 772a8fc51e25..351c4be8b2d4 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py @@ -4,6 +4,7 @@ import datetime import re +from dataclasses import InitVar, dataclass, field from typing import Any, Iterable, Mapping, Optional import dateutil @@ -14,9 +15,11 @@ from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin -class DatetimeStreamSlicer(StreamSlicer): +@dataclass +class DatetimeStreamSlicer(StreamSlicer, JsonSchemaMixin): """ Slices the stream over a datetime range. @@ -31,70 +34,60 @@ class DatetimeStreamSlicer(StreamSlicer): - days, d For example, "1d" will produce windows of 1 day, and 2weeks windows of 2 weeks. + + Attributes: + start_datetime (MinMaxDatetime): the datetime that determines the earliest record that should be synced + end_datetime (MinMaxDatetime): the datetime that determines the last record that should be synced + step (str): size of the timewindow + cursor_field (InterpolatedString): record's cursor field + datetime_format (str): format of the datetime + config (Config): connection config + start_time_option (Optional[RequestOption]): request option for start time + end_time_option (Optional[RequestOption]): request option for end time + stream_state_field_start (Optional[str]): stream slice start time field + stream_state_field_end (Optional[str]): stream slice end time field + lookback_window (Optional[InterpolatedString]): how many days before start_datetime to read data for """ + start_datetime: MinMaxDatetime + end_datetime: MinMaxDatetime + step: str + cursor_field: InterpolatedString + datetime_format: str + config: Config + options: InitVar[Mapping[str, Any]] + _cursor: dict = field(repr=False, default=None) # tracks current datetime + _cursor_end: dict = field(repr=False, default=None) # tracks end of current stream slice + start_time_option: Optional[RequestOption] = None + end_time_option: Optional[RequestOption] = None + stream_state_field_start: Optional[str] = None + stream_state_field_end: Optional[str] = None + lookback_window: Optional[InterpolatedString] = None + timedelta_regex = re.compile(r"((?P[\.\d]+?)w)?" r"((?P[\.\d]+?)d)?$") - def __init__( - self, - start_datetime: MinMaxDatetime, - end_datetime: MinMaxDatetime, - step: str, - cursor_field: InterpolatedString, - datetime_format: str, - config: Config, - start_time_option: Optional[RequestOption] = None, - end_time_option: Optional[RequestOption] = None, - stream_state_field_start: Optional[str] = None, - stream_state_field_end: Optional[str] = None, - lookback_window: Optional[InterpolatedString] = None, - **options: Optional[Mapping[str, Any]], - ): - """ - :param start_datetime: - :param end_datetime: - :param step: size of the timewindow - :param cursor_field: record's cursor field - :param datetime_format: format of the datetime - :param config: connection config - :param start_time_option: request option for start time - :param end_time_option: request option for end time - :param stream_state_field_start: stream slice start time field - :param stream_state_field_end: stream slice end time field - :param lookback_window: how many days before start_datetime to read data for - :param options: Additional runtime parameters to be used for string interpolation - """ + def __post_init__(self, options: Mapping[str, Any]): self._timezone = datetime.timezone.utc self._interpolation = JinjaInterpolation() - self._datetime_format = datetime_format - self._start_datetime = start_datetime - self._end_datetime = end_datetime - self._step = self._parse_timedelta(step) - self._config = config - self._cursor_field = InterpolatedString.create(cursor_field, options=options) - self._start_time_option = start_time_option - self._end_time_option = end_time_option - self._stream_slice_field_start = InterpolatedString.create(stream_state_field_start or "start_time", options=options) - self._stream_slice_field_end = InterpolatedString.create(stream_state_field_end or "end_time", options=options) - self._cursor = None # tracks current datetime - self._cursor_end = None # tracks end of current stream slice - self._lookback_window = lookback_window - self._options = options + self._step = self._parse_timedelta(self.step) + self.cursor_field = InterpolatedString.create(self.cursor_field, options=options) + self.stream_slice_field_start = InterpolatedString.create(self.stream_state_field_start or "start_time", options=options) + self.stream_slice_field_end = InterpolatedString.create(self.stream_state_field_end or "end_time", options=options) # If datetime format is not specified then start/end datetime should inherit it from the stream slicer - if not self._start_datetime.datetime_format: - self._start_datetime.datetime_format = self._datetime_format - if not self._end_datetime.datetime_format: - self._end_datetime.datetime_format = self._datetime_format + if not self.start_datetime.datetime_format: + self.start_datetime.datetime_format = self.datetime_format + if not self.end_datetime.datetime_format: + self.end_datetime.datetime_format = self.datetime_format - if self._start_time_option and self._start_time_option.inject_into == RequestOptionType.path: + if self.start_time_option and self.start_time_option.inject_into == RequestOptionType.path: raise ValueError("Start time cannot be passed by path") - if self._end_time_option and self._end_time_option.inject_into == RequestOptionType.path: + if self.end_time_option and self.end_time_option.inject_into == RequestOptionType.path: raise ValueError("End time cannot be passed by path") def get_stream_state(self) -> StreamState: - return {self._cursor_field.eval(self._config): self._cursor} if self._cursor else {} + return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {} def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): """ @@ -105,9 +98,9 @@ def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] :param last_record: last record read :return: None """ - stream_slice_value = stream_slice.get(self._cursor_field.eval(self._config)) - stream_slice_value_end = stream_slice.get(self._stream_slice_field_end.eval(self._config)) - last_record_value = last_record.get(self._cursor_field.eval(self._config)) if last_record else None + stream_slice_value = stream_slice.get(self.cursor_field.eval(self.config)) + stream_slice_value_end = stream_slice.get(self.stream_slice_field_end.eval(self.config)) + last_record_value = last_record.get(self.cursor_field.eval(self.config)) if last_record else None cursor = None if stream_slice_value and last_record_value: cursor = max(stream_slice_value, last_record_value) @@ -119,7 +112,7 @@ def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] self._cursor = max(cursor, self._cursor) elif cursor: self._cursor = cursor - if self._stream_slice_field_end: + if self.stream_slice_field_end: self._cursor_end = stream_slice_value_end def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: @@ -135,18 +128,18 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> """ stream_state = stream_state or {} kwargs = {"stream_state": stream_state} - end_datetime = min(self._end_datetime.get_datetime(self._config, **kwargs), datetime.datetime.now(tz=datetime.timezone.utc)) - lookback_delta = self._parse_timedelta(self._lookback_window.eval(self._config, **kwargs) if self._lookback_window else "0d") - start_datetime = self._start_datetime.get_datetime(self._config, **kwargs) - lookback_delta + end_datetime = min(self.end_datetime.get_datetime(self.config, **kwargs), datetime.datetime.now(tz=datetime.timezone.utc)) + lookback_delta = self._parse_timedelta(self.lookback_window.eval(self.config, **kwargs) if self.lookback_window else "0d") + start_datetime = self.start_datetime.get_datetime(self.config, **kwargs) - lookback_delta start_datetime = min(start_datetime, end_datetime) - if self._cursor_field.eval(self._config, stream_state=stream_state) in stream_state: - cursor_datetime = self.parse_date(stream_state[self._cursor_field.eval(self._config)]) + if self.cursor_field.eval(self.config, stream_state=stream_state) in stream_state: + cursor_datetime = self.parse_date(stream_state[self.cursor_field.eval(self.config)]) else: cursor_datetime = start_datetime start_datetime = max(cursor_datetime, start_datetime) - state_date = self.parse_date(stream_state.get(self._cursor_field.eval(self._config, stream_state=stream_state))) + state_date = self.parse_date(stream_state.get(self.cursor_field.eval(self.config, stream_state=stream_state))) if state_date: # If the input_state's date is greater than start_datetime, the start of the time window is the state's next day next_date = state_date + datetime.timedelta(days=1) @@ -155,14 +148,14 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> return dates def _format_datetime(self, dt: datetime.datetime): - if self._datetime_format == "timestamp": + if self.datetime_format == "timestamp": return dt.timestamp() else: - return dt.strftime(self._datetime_format) + return dt.strftime(self.datetime_format) def _partition_daterange(self, start, end, step: datetime.timedelta): - start_field = self._stream_slice_field_start.eval(self._config) - end_field = self._stream_slice_field_end.eval(self._config) + start_field = self.stream_slice_field_start.eval(self.config) + end_field = self.stream_slice_field_end.eval(self.config) dates = [] while start <= end: end_date = self._get_date(start + step - datetime.timedelta(days=1), end, min) @@ -206,7 +199,7 @@ def _parse_timedelta(cls, time_str): time_params = {name: float(param) for name, param in parts.groupdict().items() if param} return datetime.timedelta(**time_params) - def request_params( + def get_request_params( self, *, stream_state: Optional[StreamState] = None, @@ -215,7 +208,7 @@ def request_params( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.request_parameter, stream_slice) - def request_headers( + def get_request_headers( self, *, stream_state: Optional[StreamState] = None, @@ -224,7 +217,7 @@ def request_headers( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.header, stream_slice) - def request_body_data( + def get_request_body_data( self, *, stream_state: Optional[StreamState] = None, @@ -233,7 +226,7 @@ def request_body_data( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.body_data, stream_slice) - def request_body_json( + def get_request_body_json( self, *, stream_state: Optional[StreamState] = None, @@ -248,10 +241,8 @@ def request_kwargs(self) -> Mapping[str, Any]: def _get_request_options(self, option_type: RequestOptionType, stream_slice: StreamSlice): options = {} - if self._start_time_option and self._start_time_option.inject_into == option_type: - options[self._start_time_option.field_name] = stream_slice.get( - self._stream_slice_field_start.eval(self._config, **self._options) - ) - if self._end_time_option and self._end_time_option.inject_into == option_type: - options[self._end_time_option.field_name] = stream_slice.get(self._stream_slice_field_end.eval(self._config, **self._options)) + if self.start_time_option and self.start_time_option.inject_into == option_type: + options[self.start_time_option.field_name] = stream_slice.get(self.stream_slice_field_start.eval(self.config)) + if self.end_time_option and self.end_time_option.inject_into == option_type: + options[self.end_time_option.field_name] = stream_slice.get(self.stream_slice_field_end.eval(self.config)) return options diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py index 2dbea6841de8..ac83d1a967cf 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, Iterable, List, Mapping, Optional, Union from airbyte_cdk.models import SyncMode @@ -9,51 +10,47 @@ from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin -class ListStreamSlicer(StreamSlicer): +@dataclass +class ListStreamSlicer(StreamSlicer, JsonSchemaMixin): """ Stream slicer that iterates over the values of a list If slice_values is a string, then evaluate it as literal and assert the resulting literal is a list + + Attributes: + slice_values (Union[str, List[str]]): The values to iterate over + cursor_field (Union[InterpolatedString, str]): The name of the cursor field + config (Config): The user-provided configuration as specified by the source's spec + request_option (Optional[RequestOption]): The request option to configure the HTTP request """ - def __init__( - self, - slice_values: Union[str, List[str]], - cursor_field: Union[InterpolatedString, str], - config: Config, - request_option: Optional[RequestOption] = None, - **options: Optional[Mapping[str, Any]], - ): - """ - :param slice_values: The values to iterate over - :param cursor_field: The name of the cursor field - :param config: The user-provided configuration as specified by the source's spec - :param request_option: The request option to configure the HTTP request - :param options: Additional runtime parameters to be used for string interpolation - """ - if isinstance(slice_values, str): - slice_values = InterpolatedString.create(slice_values, options=options).eval(config) - if isinstance(cursor_field, str): - cursor_field = InterpolatedString(cursor_field, options=options) - self._cursor_field = cursor_field - self._slice_values = slice_values - self._config = config - self._cursor = None - self._request_option = request_option + slice_values: Union[str, List[str]] + cursor_field: Union[InterpolatedString, str] + config: Config + options: InitVar[Mapping[str, Any]] + request_option: Optional[RequestOption] = None - if request_option and request_option.inject_into == RequestOptionType.path: + def __post_init__(self, options: Mapping[str, Any]): + if isinstance(self.slice_values, str): + self.slice_values = InterpolatedString.create(self.slice_values, options=options).eval(self.config) + if isinstance(self.cursor_field, str): + self.cursor_field = InterpolatedString(string=self.cursor_field, options=options) + + if self.request_option and self.request_option.inject_into == RequestOptionType.path: raise ValueError("Slice value cannot be injected in the path") + self._cursor = None def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - slice_value = stream_slice.get(self._cursor_field.eval(self._config)) - if slice_value and slice_value in self._slice_values: + slice_value = stream_slice.get(self.cursor_field.eval(self.config)) + if slice_value and slice_value in self.slice_values: self._cursor = slice_value def get_stream_state(self) -> StreamState: - return {self._cursor_field.eval(self._config): self._cursor} if self._cursor else {} + return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {} - def request_params( + def get_request_params( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -61,7 +58,7 @@ def request_params( ) -> Mapping[str, Any]: return self._get_request_option(RequestOptionType.request_parameter) - def request_headers( + def get_request_headers( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -69,7 +66,7 @@ def request_headers( ) -> Mapping[str, Any]: return self._get_request_option(RequestOptionType.header) - def request_body_data( + def get_request_body_data( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -77,7 +74,7 @@ def request_body_data( ) -> Mapping[str, Any]: return self._get_request_option(RequestOptionType.body_data) - def request_body_json( + def get_request_body_json( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -86,10 +83,10 @@ def request_body_json( return self._get_request_option(RequestOptionType.body_json) def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: - return [{self._cursor_field.eval(self._config): slice_value} for slice_value in self._slice_values] + return [{self.cursor_field.eval(self.config): slice_value} for slice_value in self.slice_values] def _get_request_option(self, request_option_type: RequestOptionType): - if self._request_option and self._request_option.inject_into == request_option_type: - return {self._request_option.field_name: self._cursor} + if self.request_option and self.request_option.inject_into == request_option_type: + return {self.request_option.field_name: self._cursor} else: return {} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/single_slice.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/single_slice.py index 161cdae970ff..532982de9d08 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/single_slice.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/single_slice.py @@ -2,18 +2,20 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from dataclasses import InitVar, dataclass from typing import Any, Iterable, Mapping, Optional from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin -class SingleSlice(StreamSlicer): +@dataclass +class SingleSlice(StreamSlicer, JsonSchemaMixin): """Stream slicer returning only a single stream slice""" - def __init__(self, **options): - pass + options: InitVar[Mapping[str, Any]] def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): pass @@ -21,7 +23,7 @@ def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] def get_stream_state(self) -> StreamState: return {} - def request_params( + def get_request_params( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -29,7 +31,7 @@ def request_params( ) -> Mapping[str, Any]: return {} - def request_headers( + def get_request_headers( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -37,7 +39,7 @@ def request_headers( ) -> Mapping[str, Any]: return {} - def request_body_data( + def get_request_body_data( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -45,7 +47,7 @@ def request_body_data( ) -> Mapping[str, Any]: return {} - def request_body_json( + def get_request_body_json( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py index 025aa2bf1556..4ff22ce12c61 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py @@ -3,6 +3,7 @@ # from abc import abstractmethod +from dataclasses import dataclass from typing import Iterable, Optional from airbyte_cdk.models import SyncMode @@ -10,6 +11,7 @@ from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +@dataclass class StreamSlicer(RequestOptionsProvider): """ Slices the stream into a subset of records. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py index b387a7027509..d5b8b306b86d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py @@ -2,7 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from dataclasses import dataclass +from dataclasses import InitVar, dataclass from typing import Any, Iterable, List, Mapping, Optional from airbyte_cdk.models import SyncMode @@ -10,6 +10,7 @@ from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState from airbyte_cdk.sources.streams.core import Stream +from dataclasses_jsonschema import JsonSchemaMixin @dataclass @@ -26,35 +27,38 @@ class ParentStreamConfig: stream: Stream parent_key: str stream_slice_field: str + options: InitVar[Mapping[str, Any]] request_option: Optional[RequestOption] = None -class SubstreamSlicer(StreamSlicer): +@dataclass +class SubstreamSlicer(StreamSlicer, JsonSchemaMixin): """ Stream slicer that iterates over the parent's stream slices and records and emits slices by interpolating the slice_definition mapping Will populate the state with `parent_stream_slice` and `parent_record` so they can be accessed by other components + + Attributes: + parent_stream_configs (List[ParentStreamConfig]): parent streams to iterate over and their config """ - def __init__(self, parent_streams_configs: List[ParentStreamConfig], **options: Optional[Mapping[str, Any]]): - """ - :param parent_streams_configs: parent streams to iterate over and their config - :param options: Additional runtime parameters to be used for string interpolation - """ - if not parent_streams_configs: + parent_stream_configs: List[ParentStreamConfig] + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): + if not self.parent_stream_configs: raise ValueError("SubstreamSlicer needs at least 1 parent stream") - self._parent_stream_configs = parent_streams_configs self._cursor = None self._options = options def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): cursor = {} - for parent_stream_config in self._parent_stream_configs: + for parent_stream_config in self.parent_stream_configs: slice_value = stream_slice.get(parent_stream_config.stream_slice_field) if slice_value: cursor.update({parent_stream_config.stream_slice_field: slice_value}) self._cursor = cursor - def request_params( + def get_request_params( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -62,7 +66,7 @@ def request_params( ) -> Mapping[str, Any]: return self._get_request_option(RequestOptionType.request_parameter) - def request_headers( + def get_request_headers( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -70,7 +74,7 @@ def request_headers( ) -> Mapping[str, Any]: return self._get_request_option(RequestOptionType.header) - def request_body_data( + def get_request_body_data( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -78,7 +82,7 @@ def request_body_data( ) -> Mapping[str, Any]: return self._get_request_option(RequestOptionType.body_data) - def request_body_json( + def get_request_body_json( self, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, @@ -88,7 +92,7 @@ def request_body_json( def _get_request_option(self, option_type: RequestOptionType): params = {} - for parent_config in self._parent_stream_configs: + for parent_config in self.parent_stream_configs: if parent_config.request_option and parent_config.request_option.inject_into == option_type: key = parent_config.stream_slice_field value = self._cursor.get(key) @@ -114,10 +118,10 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Itera - parent_record: mapping representing the parent record - parent_stream_name: string representing the parent stream name """ - if not self._parent_stream_configs: + if not self.parent_stream_configs: yield from [] else: - for parent_stream_config in self._parent_stream_configs: + for parent_stream_config in self.parent_stream_configs: parent_stream = parent_stream_config.stream parent_field = parent_stream_config.parent_key stream_state_field = parent_stream_config.stream_slice_field diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py index d28e0941fc26..51ed5468acbd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py @@ -2,13 +2,14 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from dataclasses import dataclass +from dataclasses import InitVar, dataclass, field from typing import Any, List, Mapping, Optional, Union import dpath.util from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, FieldPointer, Record, StreamSlice, StreamState +from dataclasses_jsonschema import JsonSchemaMixin @dataclass(frozen=True) @@ -17,6 +18,7 @@ class AddedFieldDefinition: path: FieldPointer value: Union[InterpolatedString, str] + options: InitVar[Mapping[str, Any]] @dataclass(frozen=True) @@ -25,9 +27,11 @@ class ParsedAddFieldDefinition: path: FieldPointer value: InterpolatedString + options: InitVar[Mapping[str, Any]] -class AddFields(RecordTransformation): +@dataclass +class AddFields(RecordTransformation, JsonSchemaMixin): """ Transformation which adds field to an output record. The path of the added field can be nested. Adding nested fields will create all necessary parent objects (like mkdir -p). Adding fields to an array will extend the array to that index (filling intermediate @@ -73,25 +77,31 @@ class AddFields(RecordTransformation): # by supplying any valid Jinja template directive or expression https://jinja.palletsprojects.com/en/3.1.x/templates/# - path: ["two_times_two"] value: {{ 2 * 2 }} + + Attributes: + fields (List[AddedFieldDefinition]): A list of transformations (path and corresponding value) that will be added to the record """ - def __init__(self, fields: List[AddedFieldDefinition], **options: Optional[Mapping[str, Any]]): - """ - :param fields: Fields to add - :param options: Additional runtime parameters to be used for string interpolation - """ - self._fields: List[ParsedAddFieldDefinition] = [] - for field in fields: - if len(field.path) < 1: - raise f"Expected a non-zero-length path for the AddFields transformation {field}" - - if not isinstance(field.value, InterpolatedString): - if not isinstance(field.value, str): - raise f"Expected a string value for the AddFields transformation: {field}" + fields: List[AddedFieldDefinition] + options: InitVar[Mapping[str, Any]] + _parsed_fields: List[ParsedAddFieldDefinition] = field(init=False, repr=False, default_factory=list) + + def __post_init__(self, options: Mapping[str, Any]): + for add_field in self.fields: + if len(add_field.path) < 1: + raise f"Expected a non-zero-length path for the AddFields transformation {add_field}" + + if not isinstance(add_field.value, InterpolatedString): + if not isinstance(add_field.value, str): + raise f"Expected a string value for the AddFields transformation: {add_field}" else: - self._fields.append(ParsedAddFieldDefinition(field.path, InterpolatedString.create(field.value, options=options))) + self._parsed_fields.append( + ParsedAddFieldDefinition( + add_field.path, InterpolatedString.create(add_field.value, options=options), options=options + ) + ) else: - self._fields.append(ParsedAddFieldDefinition(field.path, field.value)) + self._parsed_fields.append(ParsedAddFieldDefinition(add_field.path, add_field.value, options={})) def transform( self, @@ -101,9 +111,9 @@ def transform( stream_slice: Optional[StreamSlice] = None, ) -> Record: kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice} - for field in self._fields: - value = field.value.eval(config, **kwargs) - dpath.util.new(record, field.path, value) + for parsed_field in self._parsed_fields: + value = parsed_field.value.eval(config, **kwargs) + dpath.util.new(record, parsed_field.path, value) return record diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py index 792c39e95fc1..7c568a45941e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py @@ -2,15 +2,18 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import List +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping import dpath.exceptions import dpath.util from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import FieldPointer, Record +from dataclasses_jsonschema import JsonSchemaMixin -class RemoveFields(RecordTransformation): +@dataclass +class RemoveFields(RecordTransformation, JsonSchemaMixin): """ A transformation which removes fields from a record. The fields removed are designated using FieldPointers. During transformation, if a field or any of its parents does not exist in the record, no error is thrown. @@ -31,20 +34,20 @@ class RemoveFields(RecordTransformation): - ["path", "to", "field1"] - ["path2"] ``` + + Attributes: + field_pointers (List[FieldPointer]): pointers to the fields that should be removed """ - def __init__(self, field_pointers: List[FieldPointer]): - """ - :param field_pointers: pointers to the fields that should be removed - """ - self._field_pointers = field_pointers + field_pointers: List[FieldPointer] + options: InitVar[Mapping[str, Any]] def transform(self, record: Record, **kwargs) -> Record: """ :param record: The record to be transformed :return: the input record with the requested fields removed """ - for pointer in self._field_pointers: + for pointer in self.field_pointers: # the dpath library by default doesn't delete fields from arrays try: dpath.util.delete(record, pointer) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py index f7c0d8c9ce6d..1b2c429687d0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py @@ -3,11 +3,13 @@ # from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Optional from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +@dataclass class RecordTransformation(ABC): """ Implementations of this class define transformations that can be applied to records of a stream. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index cbad72a4f3a8..2a8bd7283371 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -32,15 +32,15 @@ def get_access_token(self) -> str: t0 = pendulum.now() token, expires_in = self.refresh_access_token() self.access_token = token - self.token_expiry_date = t0.add(seconds=expires_in) + self.set_token_expiry_date(t0.add(seconds=expires_in)) return self.access_token def token_has_expired(self) -> bool: """Returns True if the token is expired""" - return pendulum.now() > self.token_expiry_date + return pendulum.now() > self.get_token_expiry_date() - def get_refresh_request_body(self) -> Mapping[str, Any]: + def build_refresh_request_body(self) -> Mapping[str, Any]: """ Returns the request body to set on the refresh request @@ -48,16 +48,16 @@ def get_refresh_request_body(self) -> Mapping[str, Any]: """ payload: MutableMapping[str, Any] = { "grant_type": "refresh_token", - "client_id": self.client_id, - "client_secret": self.client_secret, - "refresh_token": self.refresh_token, + "client_id": self.get_client_id(), + "client_secret": self.get_client_secret(), + "refresh_token": self.get_refresh_token(), } - if self.scopes: - payload["scopes"] = self.scopes + if self.get_scopes: + payload["scopes"] = self.get_scopes() - if self.refresh_request_body: - for key, val in self.refresh_request_body.items(): + if self.get_refresh_request_body(): + for key, val in self.get_refresh_request_body().items(): # We defer to existing oauth constructs over custom configured fields if key not in payload: payload[key] = val @@ -71,61 +71,51 @@ def refresh_access_token(self) -> Tuple[str, int]: :return: a tuple of (access_token, token_lifespan_in_seconds) """ try: - response = requests.request(method="POST", url=self.token_refresh_endpoint, data=self.get_refresh_request_body()) + response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body()) response.raise_for_status() response_json = response.json() - return response_json[self.access_token_name], response_json[self.expires_in_name] + return response_json[self.get_access_token_name()], response_json[self.get_expires_in_name()] except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e - @property @abstractmethod - def token_refresh_endpoint(self) -> str: + def get_token_refresh_endpoint(self) -> str: """Returns the endpoint to refresh the access token""" - @property @abstractmethod - def client_id(self) -> str: + def get_client_id(self) -> str: """The client id to authenticate""" - @property @abstractmethod - def client_secret(self) -> str: + def get_client_secret(self) -> str: """The client secret to authenticate""" - @property @abstractmethod - def refresh_token(self) -> str: + def get_refresh_token(self) -> str: """The token used to refresh the access token when it expires""" - @property @abstractmethod - def scopes(self) -> List[str]: + def get_scopes(self) -> List[str]: """List of requested scopes""" - @property @abstractmethod - def token_expiry_date(self) -> pendulum.datetime: + def get_token_expiry_date(self) -> pendulum.datetime: """Expiration date of the access token""" - @token_expiry_date.setter @abstractmethod - def token_expiry_date(self, value: pendulum.datetime): + def set_token_expiry_date(self, value: pendulum.datetime): """Setter for access token expiration date""" - @property @abstractmethod - def access_token_name(self) -> str: + def get_access_token_name(self) -> str: """Field to extract access token from in the response""" - @property @abstractmethod - def expires_in_name(self): - """Setter for field to extract access token expiration date from in the response""" + def get_expires_in_name(self) -> str: + """Returns the expires_in field name""" - @property @abstractmethod - def refresh_request_body(self) -> Mapping[str, Any]: + def get_refresh_request_body(self) -> Mapping[str, Any]: """Returns the request body to set on the refresh request""" @property diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index ec37f436b6e7..d479652f78b8 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -26,89 +26,47 @@ def __init__( expires_in_name: str = "expires_in", refresh_request_body: Mapping[str, Any] = None, ): - self.token_refresh_endpoint = token_refresh_endpoint - self.client_secret = client_secret - self.client_id = client_id - self.refresh_token = refresh_token - self.scopes = scopes - self.access_token_name = access_token_name - self.expires_in_name = expires_in_name - self.refresh_request_body = refresh_request_body - - self.token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) - self.access_token = None - - @property - def token_refresh_endpoint(self) -> str: + self._token_refresh_endpoint = token_refresh_endpoint + self._client_secret = client_secret + self._client_id = client_id + self._refresh_token = refresh_token + self._scopes = scopes + self._access_token_name = access_token_name + self._expires_in_name = expires_in_name + self._refresh_request_body = refresh_request_body + + self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) + self._access_token = None + + def get_token_refresh_endpoint(self) -> str: return self._token_refresh_endpoint - @token_refresh_endpoint.setter - def token_refresh_endpoint(self, value: str): - self._token_refresh_endpoint = value - - @property - def client_id(self) -> str: + def get_client_id(self) -> str: return self._client_id - @client_id.setter - def client_id(self, value: str): - self._client_id = value - - @property - def client_secret(self) -> str: + def get_client_secret(self) -> str: return self._client_secret - @client_secret.setter - def client_secret(self, value: str): - self._client_secret = value - - @property - def refresh_token(self) -> str: + def get_refresh_token(self) -> str: return self._refresh_token - @refresh_token.setter - def refresh_token(self, value: str): - self._refresh_token = value - - @property - def access_token_name(self) -> str: + def get_access_token_name(self) -> str: return self._access_token_name - @access_token_name.setter - def access_token_name(self, value: str): - self._access_token_name = value - - @property - def scopes(self) -> [str]: + def get_scopes(self) -> [str]: return self._scopes - @scopes.setter - def scopes(self, value: [str]): - self._scopes = value - - @property - def token_expiry_date(self) -> pendulum.DateTime: - return self._token_expiry_date - - @token_expiry_date.setter - def token_expiry_date(self, value: pendulum.DateTime): - self._token_expiry_date = value - - @property - def expires_in_name(self) -> str: + def get_expires_in_name(self) -> str: return self._expires_in_name - @expires_in_name.setter - def expires_in_name(self, value): - self._expires_in_name = value - - @property - def refresh_request_body(self) -> Mapping[str, Any]: + def get_refresh_request_body(self) -> Mapping[str, Any]: return self._refresh_request_body - @refresh_request_body.setter - def refresh_request_body(self, value: Mapping[str, Any]): - self._refresh_request_body = value + def get_token_expiry_date(self) -> pendulum.DateTime: + return self._token_expiry_date + + def set_token_expiry_date(self, value: pendulum.DateTime): + self._token_expiry_date = value @property def access_token(self) -> str: diff --git a/airbyte-cdk/python/reference_docs/_source/conf.py b/airbyte-cdk/python/reference_docs/_source/conf.py index ff5dcf2caec5..5ce9636934f6 100644 --- a/airbyte-cdk/python/reference_docs/_source/conf.py +++ b/airbyte-cdk/python/reference_docs/_source/conf.py @@ -32,7 +32,10 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.autodoc"] # API docs +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", # Support for NumPy and Google style docstrings +] # API docs source_suffix = {".rst": "restructuredtext", ".md": "markdown"} diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index d314389c93d5..22643e5fced3 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -44,6 +44,7 @@ packages=find_packages(exclude=("unit_tests",)), install_requires=[ "backoff", + "dataclasses-jsonschema~=2.15.1", "dpath~=2.0.1", "jsonschema~=3.2.0", "jsonref~=0.2", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py index 2d0c1d265fa8..12cb353de5c0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py @@ -17,11 +17,11 @@ "refresh_endpoint": "refresh_end", "client_id": "some_client_id", "client_secret": "some_client_secret", - "refresh_token": "some_refresh_token", "token_expiry_date": pendulum.now().subtract(days=2).to_rfc3339_string(), "custom_field": "in_outbound_request", "another_field": "exists_in_body", } +options = {"refresh_token": "some_refresh_token"} class TestOauth2Authenticator: @@ -38,7 +38,7 @@ def test_refresh_request_body(self): token_refresh_endpoint="{{ config['refresh_endpoint'] }}", client_id="{{ config['client_id'] }}", client_secret="{{ config['client_secret'] }}", - refresh_token="{{ config['refresh_token'] }}", + refresh_token="{{ options['refresh_token'] }}", config=config, scopes=["scope1", "scope2"], token_expiry_date="{{ config['token_expiry_date'] }}", @@ -47,8 +47,9 @@ def test_refresh_request_body(self): "another_field": "{{ config['another_field'] }}", "scopes": ["no_override"], }, + options=options, ) - body = oauth.get_refresh_request_body() + body = oauth.build_refresh_request_body() expected = { "grant_type": "refresh_token", "client_id": "some_client_id", @@ -74,6 +75,7 @@ def test_refresh_access_token(self, mocker): "another_field": "{{ config['another_field'] }}", "scopes": ["no_override"], }, + options={}, ) resp.status_code = 200 @@ -81,6 +83,9 @@ def test_refresh_access_token(self, mocker): mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True) token = oauth.refresh_access_token() + schem = DeclarativeOauth2Authenticator.json_schema() + print(schem) + assert ("access_token", 1000) == token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py index b46f9eac643e..29613a73fbe5 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py @@ -13,6 +13,7 @@ resp = Response() config = {"username": "user", "password": "password", "header": "header"} +options = {"username": "user", "password": "password", "header": "header"} @pytest.mark.parametrize( @@ -20,13 +21,14 @@ [ ("test_static_token", "test-token", "Bearer test-token"), ("test_token_from_config", "{{ config.username }}", "Bearer user"), + ("test_token_from_options", "{{ options.username }}", "Bearer user"), ], ) def test_bearer_token_authenticator(test_name, token, expected_header_value): """ Should match passed in token, no matter how many times token is retrieved. """ - token_auth = BearerAuthenticator(token, config) + token_auth = BearerAuthenticator(token, config, options=options) header1 = token_auth.get_auth_header() header2 = token_auth.get_auth_header() @@ -44,13 +46,14 @@ def test_bearer_token_authenticator(test_name, token, expected_header_value): [ ("test_static_creds", "user", "password", "Basic dXNlcjpwYXNzd29yZA=="), ("test_creds_from_config", "{{ config.username }}", "{{ config.password }}", "Basic dXNlcjpwYXNzd29yZA=="), + ("test_creds_from_options", "{{ options.username }}", "{{ options.password }}", "Basic dXNlcjpwYXNzd29yZA=="), ], ) def test_basic_authenticator(test_name, username, password, expected_header_value): """ Should match passed in token, no matter how many times token is retrieved. """ - token_auth = BasicHttpAuthenticator(username=username, password=password, config=config) + token_auth = BasicHttpAuthenticator(username=username, password=password, config=config, options=options) header1 = token_auth.get_auth_header() header2 = token_auth.get_auth_header() @@ -68,13 +71,14 @@ def test_basic_authenticator(test_name, username, password, expected_header_valu [ ("test_static_token", "Authorization", "test-token", "Authorization", "test-token"), ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user"), + ("test_token_from_options", "{{ options.header }}", "{{ options.username }}", "header", "user"), ], ) def test_api_key_authenticator(test_name, header, token, expected_header, expected_header_value): """ Should match passed in token, no matter how many times token is retrieved. """ - token_auth = ApiKeyAuthenticator(header, token, config) + token_auth = ApiKeyAuthenticator(header=header, api_token=token, config=config, options=options) header1 = token_auth.get_auth_header() header2 = token_auth.get_auth_header() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py index 98aa2f1bdc13..827b99ab6484 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py @@ -30,7 +30,7 @@ def test_check_stream(test_name, record, streams_to_check, expectation): source = MagicMock() source.streams.return_value = [stream] - check_stream = CheckStream(streams_to_check) + check_stream = CheckStream(streams_to_check, options={}) if expectation: actual = check_stream.check_connection(source, logger, config) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py index 56f0a69c3598..f67032c02d58 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py @@ -30,13 +30,17 @@ "{{ stream_state['newer'] }}", middle_date, ), + ("test_min_newer_time_from_options", "{{ config['older'] }}", "{{ options['newer'] }}", "", new_date), + ("test_max_newer_time_from_options", "{{ stream_state['newer'] }}", "", "{{ options['older'] }}", old_date), ], ) def test_min_max_datetime(test_name, date, min_date, max_date, expected_date): + print(MinMaxDatetime.json_schema()) config = {"older": old_date, "middle": middle_date} stream_state = {"newer": new_date} + options = {"newer": new_date, "older": old_date} - min_max_date = MinMaxDatetime(datetime=date, min_datetime=min_date, max_datetime=max_date) + min_max_date = MinMaxDatetime(datetime=date, min_datetime=min_date, max_datetime=max_date, options=options) actual_date = min_max_date.get_datetime(config, **{"stream_state": stream_state}) assert actual_date == datetime.datetime.strptime(expected_date, date_format) @@ -51,6 +55,7 @@ def test_custom_datetime_format(): datetime_format="%Y-%m-%dT%H:%M:%S", min_datetime="{{ config['older'] }}", max_datetime="{{ stream_state['newer'] }}", + options={}, ) actual_date = min_max_date.get_datetime(config, **{"stream_state": stream_state}) @@ -66,7 +71,26 @@ def test_format_is_a_number(): datetime_format="%Y%m%d", min_datetime="{{ config['older'] }}", max_datetime="{{ stream_state['newer'] }}", + options={}, ) actual_date = min_max_date.get_datetime(config, **{"stream_state": stream_state}) assert actual_date == datetime.datetime.strptime("20220101", "%Y%m%d").replace(tzinfo=datetime.timezone.utc) + + +def test_set_datetime_format(): + min_max_date = MinMaxDatetime(datetime="{{ config['middle'] }}", min_datetime="{{ config['older'] }}", options={}) + + # Retrieve datetime using the default datetime formatting + default_fmt_config = {"older": "2021-01-01T20:12:19.597854Z", "middle": "2022-01-01T20:12:19.597854Z"} + actual_date = min_max_date.get_datetime(default_fmt_config) + + assert actual_date == datetime.datetime.strptime("2022-01-01T20:12:19.597854Z", "%Y-%m-%dT%H:%M:%S.%f%z") + + # Set a different datetime format and attempt to retrieve datetime using an updated format + min_max_date.datetime_format = "%Y-%m-%dT%H:%M:%S" + + custom_fmt_config = {"older": "2021-01-01T20:12:19", "middle": "2022-01-01T20:12:19"} + actual_date = min_max_date.get_datetime(custom_fmt_config) + + assert actual_date == datetime.datetime.strptime("2022-01-01T20:12:19", "%Y-%m-%dT%H:%M:%S").replace(tzinfo=datetime.timezone.utc) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py index 6812e55be11e..b9a1ec25322d 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py @@ -10,8 +10,9 @@ from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor config = {"field": "record_array"} +options = {"options_field": "record_array"} -decoder = JsonDecoder() +decoder = JsonDecoder(options={}) @pytest.mark.parametrize( @@ -19,6 +20,7 @@ [ ("test_extract_from_array", "_.data", {"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), ("test_field_in_config", "_.{{ config['field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_field_in_options", "_.{{ options['options_field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), ("test_default", "_{{kwargs['field']}}", [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]), ( "test_remove_fields_from_records", @@ -38,7 +40,7 @@ ], ) def test(test_name, transform, body, expected_records): - extractor = JelloExtractor(transform, config, decoder) + extractor = JelloExtractor(transform=transform, config=config, decoder=decoder, options=options) response = create_response(body) actual_records = extractor.extract_records(response) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py index 2b180ee5d935..e58db11ada56 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py @@ -33,14 +33,21 @@ [{"id": 11}, {"id": 12}, {"id": 13}, {"id": 14}, {"id": 15}], [], ), + ( + "test_using_options_filter", + "{{ record['created_at'] > options['created_at'] }}", + [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}], + [{"id": 3, "created_at": "06-08-21"}], + ), ], ) def test_record_filter(test_name, filter_template, records, expected_records): config = {"response_override": "stop_if_you_see_me"} + options = {"created_at": "06-07-21"} stream_state = {"created_at": "06-06-21"} stream_slice = {"last_seen": "06-10-21"} next_page_token = {"last_seen_id": 14} - record_filter = RecordFilter(config=config, condition=filter_template) + record_filter = RecordFilter(config=config, condition=filter_template, options=options) actual_records = record_filter.filter_records( records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index 0367d7d34a18..ed7aa35e6245 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -29,22 +29,30 @@ {"data": [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}]}, [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}], ), + ( + "test_with_extractor_and_filter_with_options", + "_.{{ options['options_field'] }}", + "{{ record['created_at'] > options['created_at'] }}", + {"data": [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}]}, + [{"id": 3, "created_at": "06-08-21"}], + ), ], ) def test_record_filter(test_name, transform_template, filter_template, body, expected_records): config = {"response_override": "stop_if_you_see_me"} + options = {"options_field": "data", "created_at": "06-07-21"} stream_state = {"created_at": "06-06-21"} stream_slice = {"last_seen": "06-10-21"} next_page_token = {"last_seen_id": 14} response = create_response(body) - decoder = JsonDecoder() - extractor = JelloExtractor(transform=transform_template, decoder=decoder, config=config) + decoder = JsonDecoder(options={}) + extractor = JelloExtractor(transform=transform_template, decoder=decoder, config=config, options=options) if filter_template is None: record_filter = None else: - record_filter = RecordFilter(config=config, condition=filter_template) - record_selector = RecordSelector(extractor=extractor, record_filter=record_filter) + record_filter = RecordFilter(config=config, condition=filter_template, options=options) + record_selector = RecordSelector(extractor=extractor, record_filter=record_filter, options=options) actual_records = record_selector.select_records( response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py index adc832a7bdd5..244d041846cb 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py @@ -19,7 +19,6 @@ @pytest.mark.parametrize( "test_name, template, expected_result", [ - ("test_static_condition", True, True), ("test_interpolated_true_value", "{{ config['parent']['key_with_true'] }}", True), ("test_interpolated_true_comparison", "{{ config['string_key'] == \"compare_me\" }}", True), ("test_interpolated_false_condition", "{{ config['string_key'] == \"witness_me\" }}", False), @@ -34,8 +33,9 @@ ("test_True", "{{ True }}", True), ("test_value_in_array", "{{ 1 in config['non_empty_array'] }}", True), ("test_value_not_in_array", "{{ 2 in config['non_empty_array'] }}", False), + ("test_interpolation_using_options", "{{ options['from_options'] == \"come_find_me\" }}", True), ], ) def test_interpolated_boolean(test_name, template, expected_result): - interpolated_bool = InterpolatedBoolean(template) + interpolated_bool = InterpolatedBoolean(condition=template, options={"from_options": "come_find_me"}) assert interpolated_bool.eval(config) == expected_result diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py index 9413c79caaf8..8491cc6b9086 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py @@ -28,7 +28,7 @@ def test(test_name, key, expected_value): } config = {"c": "VALUE_FROM_CONFIG"} kwargs = {"a": "VALUE_FROM_KWARGS"} - mapping = InterpolatedMapping(d, options={"b": "VALUE_FROM_OPTIONS", "k": "key"}) + mapping = InterpolatedMapping(mapping=d, options={"b": "VALUE_FROM_OPTIONS", "k": "key"}) interpolated = mapping.eval(config, **{"kwargs": kwargs}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_string.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_string.py index b66d13ccc965..089174c82f52 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_string.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_string.py @@ -17,6 +17,7 @@ ("test_eval_from_options", "{{ options['hello'] }}", "world"), ("test_eval_from_config", "{{ config['field'] }}", "value"), ("test_eval_from_kwargs", "{{ kwargs['c'] }}", "airbyte"), + ("test_eval_from_kwargs", "{{ kwargs['c'] }}", "airbyte"), ], ) def test_interpolated_string(test_name, input_string, expected_value): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_only_once.py b/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_only_once.py index df58c545e56a..d51ca23b04e3 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_only_once.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_only_once.py @@ -7,7 +7,7 @@ def test(): - iterator = SingleSlice() + iterator = SingleSlice(options={}) stream_slices = iterator.stream_slices(SyncMode.incremental, None) assert stream_slices == [dict()] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py index 71518b09980d..d60a862770af 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py @@ -22,3 +22,10 @@ def test_exponential_backoff(test_name, attempt_count, expected_backoff_time): backoff_strategy = ExponentialBackoffStrategy(factor=5) backoff = backoff_strategy.backoff(response_mock, attempt_count) assert backoff == expected_backoff_time + + +def test_exponential_backoff_default(): + response_mock = MagicMock() + backoff_strategy = ExponentialBackoffStrategy() + backoff = backoff_strategy.backoff(response_mock, 3) + assert backoff == 40 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index 74fa5a30dc02..27b47f97368c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -96,7 +96,7 @@ def test_composite_error_handler(test_name, first_handler_behavior, second_handl second_error_handler.should_retry.return_value = second_handler_behavior second_error_handler.should_retry.return_value = second_handler_behavior retriers = [first_error_handler, second_error_handler] - retrier = CompositeErrorHandler(retriers) + retrier = CompositeErrorHandler(error_handlers=retriers, options={}) response_mock = MagicMock() response_mock.ok = first_handler_behavior == response_status.SUCCESS or second_handler_behavior == response_status.SUCCESS assert retrier.should_retry(response_mock) == expected_behavior @@ -104,7 +104,7 @@ def test_composite_error_handler(test_name, first_handler_behavior, second_handl def test_composite_error_handler_no_handlers(): try: - CompositeErrorHandler([]) + CompositeErrorHandler(error_handlers=[], options={}) assert False except ValueError: pass diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py index b7167f7dfcd8..091fc0293bf0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py @@ -59,7 +59,7 @@ "test_403_ignore_error_message", HTTPStatus.FORBIDDEN, None, - HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contain="found"), + HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contains="found", options={}), {}, response_status.IGNORE, None, @@ -68,7 +68,7 @@ "test_403_dont_ignore_error_message", HTTPStatus.FORBIDDEN, None, - HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contain="not_found"), + HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contains="not_found", options={}), {}, response_status.FAIL, None, @@ -78,7 +78,7 @@ "test_ignore_403", HTTPStatus.FORBIDDEN, None, - HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={HTTPStatus.FORBIDDEN}), + HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={HTTPStatus.FORBIDDEN}, options={}), {}, response_status.IGNORE, None, @@ -86,7 +86,7 @@ ( "test_403_with_predicate", HTTPStatus.FORBIDDEN, - HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'code' in response }}"), + HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'code' in response }}", options={}), None, {}, ResponseStatus.retry(10), @@ -95,7 +95,7 @@ ( "test_403_with_predicate", HTTPStatus.FORBIDDEN, - HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'some_absent_field' in response }}"), + HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'some_absent_field' in response }}", options={}), None, {}, response_status.FAIL, @@ -104,7 +104,7 @@ ( "test_200_fail_with_predicate", HTTPStatus.OK, - HttpResponseFilter(action=ResponseAction.FAIL, error_message_contain="found"), + HttpResponseFilter(action=ResponseAction.FAIL, error_message_contains="found", options={}), None, {}, response_status.FAIL, @@ -113,7 +113,7 @@ ( "test_retry_403", HTTPStatus.FORBIDDEN, - HttpResponseFilter(action=ResponseAction.RETRY, http_codes={HTTPStatus.FORBIDDEN}), + HttpResponseFilter(action=ResponseAction.RETRY, http_codes={HTTPStatus.FORBIDDEN}, options={}), None, {}, ResponseStatus.retry(10), @@ -127,7 +127,7 @@ def test_default_error_handler( response_mock = create_response(http_code, headers=response_headers, json_body={"code": "1000", "error": "found"}) response_mock.ok = http_code < 400 response_filters = [f for f in [retry_response_filter, ignore_response_filter] if f] - error_handler = DefaultErrorHandler(response_filters=response_filters, backoff_strategies=backoff_strategy) + error_handler = DefaultErrorHandler(response_filters=response_filters, backoff_strategies=backoff_strategy, options={}) actual_should_retry = error_handler.should_retry(response_mock) assert actual_should_retry == should_retry if should_retry.action == ResponseAction.RETRY: @@ -137,7 +137,7 @@ def test_default_error_handler( def test_default_error_handler_attempt_count_increases(): status_code = 500 response_mock = create_response(status_code) - error_handler = DefaultErrorHandler() + error_handler = DefaultErrorHandler(options={}) actual_should_retry = error_handler.should_retry(response_mock) assert actual_should_retry == ResponseStatus.retry(10) assert actual_should_retry.retry_in == 10 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index 73e1f0f5c36c..0299bd587341 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -18,15 +18,19 @@ ("test_token_from_config", "{{ config.config_key }}", None, "config_value"), ("test_token_from_last_record", "{{ last_records[-1].id }}", None, 1), ("test_token_from_response", "{{ response._metadata.content }}", None, "content_value"), + ("test_token_from_options", "{{ options.key }}", None, "value"), ("test_token_not_found", "{{ response.invalid_key }}", None, None), - ("test_static_token_with_stop_condition_false", "token", InterpolatedBoolean("{{False}}"), "token"), - ("test_static_token_with_stop_condition_true", "token", InterpolatedBoolean("{{True}}"), None), + ("test_static_token_with_stop_condition_false", "token", InterpolatedBoolean(condition="{{False}}", options={}), "token"), + ("test_static_token_with_stop_condition_true", "token", InterpolatedBoolean(condition="{{True}}", options={}), None), ], ) def test_cursor_pagination_strategy(test_name, template_string, stop_condition, expected_token): - decoder = JsonDecoder() + decoder = JsonDecoder(options={}) config = {"config_key": "config_value"} - strategy = CursorPaginationStrategy(template_string, config, stop_condition, decoder) + options = {"key": "value"} + strategy = CursorPaginationStrategy( + cursor_value=template_string, config=config, stop_condition=stop_condition, decoder=decoder, options=options + ) response = requests.Response() response.headers = {"has_more": True} diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py index 585da5eccd94..cbdae4e48531 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py @@ -17,7 +17,7 @@ [ ( "test_limit_paginator_path", - RequestOption(inject_into=RequestOptionType.path), + RequestOption(inject_into=RequestOptionType.path, options={}), None, "/next_url", {"limit": 2}, @@ -29,7 +29,7 @@ ), ( "test_limit_paginator_request_param", - RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from"), + RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", options={}), None, None, {"limit": 2, "from": "https://airbyte.io/next_url"}, @@ -41,8 +41,8 @@ ), ( "test_limit_paginator_no_token", - RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from"), - InterpolatedBoolean("{{True}}"), + RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", options={}), + InterpolatedBoolean(condition="{{True}}", options={}), None, {"limit": 2}, {}, @@ -53,7 +53,7 @@ ), ( "test_limit_paginator_cursor_header", - RequestOption(inject_into=RequestOptionType.header, field_name="from"), + RequestOption(inject_into=RequestOptionType.header, field_name="from", options={}), None, None, {"limit": 2}, @@ -65,7 +65,7 @@ ), ( "test_limit_paginator_cursor_body_data", - RequestOption(inject_into=RequestOptionType.body_data, field_name="from"), + RequestOption(inject_into=RequestOptionType.body_data, field_name="from", options={}), None, None, {"limit": 2}, @@ -77,7 +77,7 @@ ), ( "test_limit_paginator_cursor_body_json", - RequestOption(inject_into=RequestOptionType.body_json, field_name="from"), + RequestOption(inject_into=RequestOptionType.body_json, field_name="from", options={}), None, None, {"limit": 2}, @@ -101,12 +101,23 @@ def test_limit_paginator( last_records, expected_next_page_token, ): - limit_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit") + limit_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={}) cursor_value = "{{ response.next }}" url_base = "https://airbyte.io" config = {} - strategy = CursorPaginationStrategy(cursor_value, stop_condition=stop_condition, decoder=JsonDecoder(), config=config) - paginator = LimitPaginator(2, limit_request_option, page_token_request_option, strategy, config, url_base) + options = {} + strategy = CursorPaginationStrategy( + cursor_value=cursor_value, stop_condition=stop_condition, decoder=JsonDecoder(options={}), config=config, options=options + ) + paginator = LimitPaginator( + page_size=2, + limit_option=limit_request_option, + page_token_option=page_token_request_option, + pagination_strategy=strategy, + config=config, + url_base=url_base, + options={}, + ) response = requests.Response() response.headers = {"A_HEADER": "HEADER_VALUE"} @@ -115,10 +126,10 @@ def test_limit_paginator( actual_next_page_token = paginator.next_page_token(response, last_records) actual_next_path = paginator.path() - actual_request_params = paginator.request_params() - actual_headers = paginator.request_headers() - actual_body_data = paginator.request_body_data() - actual_body_json = paginator.request_body_json() + actual_request_params = paginator.get_request_params() + actual_headers = paginator.get_request_headers() + actual_body_data = paginator.get_request_body_data() + actual_body_json = paginator.get_request_body_json() assert actual_next_page_token == expected_next_page_token assert actual_next_path == expected_updated_path assert actual_request_params == expected_request_params @@ -128,14 +139,23 @@ def test_limit_paginator( def test_limit_cannot_be_set_in_path(): - limit_request_option = RequestOption(inject_into=RequestOptionType.path) - page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset") + limit_request_option = RequestOption(inject_into=RequestOptionType.path, options={}) + page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) cursor_value = "{{ response.next }}" url_base = "https://airbyte.io" config = {} - strategy = CursorPaginationStrategy(cursor_value, config) + options = {} + strategy = CursorPaginationStrategy(cursor_value=cursor_value, config=config, options=options) try: - LimitPaginator(2, limit_request_option, page_token_request_option, strategy, config, url_base) + LimitPaginator( + page_size=2, + limit_option=limit_request_option, + page_token_option=page_token_request_option, + pagination_strategy=strategy, + config=config, + url_base=url_base, + options={}, + ) assert False except ValueError: pass diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py index b9fcd7af21fc..637bebb8f910 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py @@ -7,6 +7,6 @@ def test(): - paginator = NoPagination() + paginator = NoPagination(options={}) next_page_token = paginator.next_page_token(requests.Response(), []) assert next_page_token == {} diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py index 866ae756427e..7376ef155b43 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py @@ -17,7 +17,7 @@ ], ) def test_offset_increment_paginator_strategy(test_name, page_size, expected_next_page_token, expected_offset): - paginator_strategy = OffsetIncrement(page_size) + paginator_strategy = OffsetIncrement(page_size, options={}) assert paginator_strategy._offset == 0 response = requests.Response() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index 7d50dfb105aa..fa3808a916b0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -17,7 +17,7 @@ ], ) def test_page_increment_paginator_strategy(test_name, page_size, expected_next_page_token, expected_offset): - paginator_strategy = PageIncrement(page_size) + paginator_strategy = PageIncrement(page_size, options={}) assert paginator_strategy._offset == 0 response = requests.Response() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py index 0ccedc6b4d14..c54be6223be8 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_request_option.py @@ -23,11 +23,11 @@ ) def test_request_option(test_name, option_type, field_name, should_raise): try: - request_option = RequestOption(inject_into=option_type, field_name=field_name) + request_option = RequestOption(inject_into=option_type, field_name=field_name, options={}) if should_raise: assert False - assert request_option._field_name == field_name - assert request_option._option_type == option_type + assert request_option.field_name == field_name + assert request_option.inject_into == option_type except ValueError: if not should_raise: assert False diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py index 65f458fecaf3..457ddc9a22d8 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py @@ -30,9 +30,9 @@ ], ) def test_interpolated_request_params(test_name, input_request_params, expected_request_params): - provider = InterpolatedRequestOptionsProvider(config=config, request_parameters=input_request_params) + provider = InterpolatedRequestOptionsProvider(config=config, request_parameters=input_request_params, options={}) - actual_request_params = provider.request_params(stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token) + actual_request_params = provider.get_request_params(stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token) assert actual_request_params == expected_request_params @@ -54,9 +54,9 @@ def test_interpolated_request_params(test_name, input_request_params, expected_r ], ) def test_interpolated_request_json(test_name, input_request_json, expected_request_json): - provider = InterpolatedRequestOptionsProvider(config=config, request_body_json=input_request_json) + provider = InterpolatedRequestOptionsProvider(config=config, request_body_json=input_request_json, options={}) - actual_request_json = provider.request_body_json(stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token) + actual_request_json = provider.get_request_body_json(stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token) assert actual_request_json == expected_request_json @@ -72,9 +72,9 @@ def test_interpolated_request_json(test_name, input_request_json, expected_reque ], ) def test_interpolated_request_data(test_name, input_request_data, expected_request_data): - provider = InterpolatedRequestOptionsProvider(config=config, request_body_data=input_request_data) + provider = InterpolatedRequestOptionsProvider(config=config, request_body_data=input_request_data, options={}) - actual_request_data = provider.request_body_data(stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token) + actual_request_data = provider.get_request_body_data(stream_state=state, stream_slice=stream_slice, next_page_token=next_page_token) assert actual_request_data == expected_request_data @@ -83,4 +83,4 @@ def test_error_on_create_for_both_request_json_and_data(): request_json = {"body_key": "{{ stream_slice['start_date'] }}"} request_data = "interpolate_me=5&invalid={{ config['option'] }}" with pytest.raises(ValueError): - InterpolatedRequestOptionsProvider(config=config, request_body_json=request_json, request_body_data=request_data) + InterpolatedRequestOptionsProvider(config=config, request_body_json=request_json, request_body_data=request_data, options={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index 0b5aabda8ca8..0a6c6b3d72c1 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -16,13 +16,13 @@ def test_http_requester(): request_params = {"param": "value"} request_body_data = "body_key_1=value_1&body_key_2=value2" request_body_json = {"body_field": "body_value"} - request_options_provider.request_params.return_value = request_params - request_options_provider.request_body_data.return_value = request_body_data - request_options_provider.request_body_json.return_value = request_body_json + request_options_provider.get_request_params.return_value = request_params + request_options_provider.get_request_body_data.return_value = request_body_data + request_options_provider.get_request_body_json.return_value = request_body_json request_headers_provider = MagicMock() request_headers = {"header": "value"} - request_headers_provider.request_headers.return_value = request_headers + request_headers_provider.get_request_headers.return_value = request_headers authenticator = MagicMock() @@ -48,14 +48,15 @@ def test_http_requester(): authenticator=authenticator, error_handler=error_handler, config=config, + options={}, ) assert requester.get_url_base() == "https://airbyte.io" assert requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={}) == "v1/1234" assert requester.get_authenticator() == authenticator assert requester.get_method() == HttpMethod.GET - assert requester.request_params(stream_state={}, stream_slice=None, next_page_token=None) == request_params - assert requester.request_body_data(stream_state={}, stream_slice=None, next_page_token=None) == request_body_data - assert requester.request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json + assert requester.get_request_params(stream_state={}, stream_slice=None, next_page_token=None) == request_params + assert requester.get_request_body_data(stream_state={}, stream_slice=None, next_page_token=None) == request_body_data + assert requester.get_request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json assert requester.should_retry(requests.Response()) == should_retry assert {} == requester.request_kwargs(stream_state={}, stream_slice=None, next_page_token=None) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py index 336248546546..74ee47267c35 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py @@ -4,7 +4,7 @@ import pytest as pytest from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping -from airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider import InterpolatedRequestInputProvider +from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider @pytest.mark.parametrize( diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index aa3b1ee215e6..ad1ce696acb5 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever +from airbyte_cdk.sources.streams.http.auth import NoAuth primary_key = "pk" records = [{"id": 1}, {"id": 2}] @@ -23,7 +24,7 @@ def test_simple_retriever_full(): requester = MagicMock() request_params = {"param": "value"} - requester.request_params.return_value = request_params + requester.get_request_params.return_value = request_params paginator = MagicMock() next_page_token = {"cursor": "cursor_value"} @@ -42,6 +43,7 @@ def test_simple_retriever_full(): underlying_state = {"date": "2021-01-01"} iterator.get_stream_state.return_value = underlying_state + requester.get_authenticator.return_value = NoAuth url_base = "https://airbyte.io" requester.get_url_base.return_value = url_base path = "/v1" @@ -52,9 +54,9 @@ def test_simple_retriever_full(): should_retry = ResponseStatus.retry(backoff_time) requester.should_retry.return_value = should_retry request_body_data = {"body": "data"} - requester.request_body_data.return_value = request_body_data + requester.get_request_body_data.return_value = request_body_data request_body_json = {"body": "json"} - requester.request_body_json.return_value = request_body_json + requester.get_request_body_json.return_value = request_body_json request_kwargs = {"kwarg": "value"} requester.request_kwargs.return_value = request_kwargs cache_filename = "cache" @@ -63,12 +65,13 @@ def test_simple_retriever_full(): requester.use_cache = use_cache retriever = SimpleRetriever( - "stream_name", - primary_key, + name="stream_name", + primary_key=primary_key, requester=requester, paginator=paginator, record_selector=record_selector, stream_slicer=iterator, + options={}, ) assert retriever.primary_key == primary_key @@ -106,7 +109,7 @@ def test_simple_retriever_full(): ) def test_should_retry(test_name, requester_response, expected_should_retry, expected_backoff_time): requester = MagicMock() - retriever = SimpleRetriever("stream_name", primary_key, requester=requester, record_selector=MagicMock()) + retriever = SimpleRetriever(name="stream_name", primary_key=primary_key, requester=requester, record_selector=MagicMock(), options={}) requester.should_retry.return_value = requester_response assert retriever.should_retry(requests.Response()) == expected_should_retry if requester_response.action == ResponseAction.RETRY: @@ -125,7 +128,9 @@ def test_parse_response(test_name, status_code, response_status, len_expected_re requester = MagicMock() record_selector = MagicMock() record_selector.select_records.return_value = [{"id": 100}] - retriever = SimpleRetriever("stream_name", primary_key, requester=requester, record_selector=record_selector) + retriever = SimpleRetriever( + name="stream_name", primary_key=primary_key, requester=requester, record_selector=record_selector, options={} + ) response = requests.Response() response.status_code = status_code requester.should_retry.return_value = response_status @@ -154,7 +159,9 @@ def test_backoff_time(test_name, response_action, retry_in, expected_backoff_tim record_selector = MagicMock() record_selector.select_records.return_value = [{"id": 100}] response = requests.Response() - retriever = SimpleRetriever("stream_name", primary_key, requester=requester, record_selector=record_selector) + retriever = SimpleRetriever( + name="stream_name", primary_key=primary_key, requester=requester, record_selector=record_selector, options={} + ) if expected_backoff_time: requester.should_retry.return_value = ResponseStatus(response_action, retry_in) actual_backoff_time = retriever.backoff_time(response) @@ -180,27 +187,33 @@ def test_backoff_time(test_name, response_action, retry_in, expected_backoff_tim ) def test_get_request_options_from_pagination(test_name, paginator_mapping, stream_slicer_mapping, expected_mapping): paginator = MagicMock() - paginator.request_headers.return_value = paginator_mapping - paginator.request_params.return_value = paginator_mapping - paginator.request_body_data.return_value = paginator_mapping - paginator.request_body_json.return_value = paginator_mapping + paginator.get_request_headers.return_value = paginator_mapping + paginator.get_request_params.return_value = paginator_mapping + paginator.get_request_body_data.return_value = paginator_mapping + paginator.get_request_body_json.return_value = paginator_mapping stream_slicer = MagicMock() - stream_slicer.request_headers.return_value = stream_slicer_mapping - stream_slicer.request_params.return_value = stream_slicer_mapping - stream_slicer.request_body_data.return_value = stream_slicer_mapping - stream_slicer.request_body_json.return_value = stream_slicer_mapping + stream_slicer.get_request_headers.return_value = stream_slicer_mapping + stream_slicer.get_request_params.return_value = stream_slicer_mapping + stream_slicer.get_request_body_data.return_value = stream_slicer_mapping + stream_slicer.get_request_body_json.return_value = stream_slicer_mapping base_mapping = {"key": "value"} requester = MagicMock() - requester.request_headers.return_value = base_mapping - requester.request_params.return_value = base_mapping - requester.request_body_data.return_value = base_mapping - requester.request_body_json.return_value = base_mapping + requester.get_request_headers.return_value = base_mapping + requester.get_request_params.return_value = base_mapping + requester.get_request_body_data.return_value = base_mapping + requester.get_request_body_json.return_value = base_mapping record_selector = MagicMock() retriever = SimpleRetriever( - "stream_name", primary_key, requester=requester, record_selector=record_selector, paginator=paginator, stream_slicer=stream_slicer + name="stream_name", + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + stream_slicer=stream_slicer, + options={}, ) request_option_type_to_method = { @@ -234,13 +247,20 @@ def test_get_request_options_from_pagination(test_name, paginator_mapping, strea ) def test_request_body_data(test_name, requester_body_data, paginator_body_data, expected_body_data): paginator = MagicMock() - paginator.request_body_data.return_value = paginator_body_data + paginator.get_request_body_data.return_value = paginator_body_data requester = MagicMock() - requester.request_body_data.return_value = requester_body_data + requester.get_request_body_data.return_value = requester_body_data record_selector = MagicMock() - retriever = SimpleRetriever("stream_name", primary_key, requester=requester, record_selector=record_selector, paginator=paginator) + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + options={}, + ) if expected_body_data: actual_body_data = retriever.request_body_data(None, None, None) @@ -268,7 +288,14 @@ def test_path(test_name, requester_path, paginator_path, expected_path): requester.get_path.return_value = requester_path record_selector = MagicMock() - retriever = SimpleRetriever("stream_name", primary_key, requester=requester, record_selector=record_selector, paginator=paginator) + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + options={}, + ) actual_path = retriever.path(stream_state=None, stream_slice=None, next_page_token=None) assert expected_path == actual_path diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py index 28563fbdaa9d..3ed21485c3c0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py @@ -17,14 +17,14 @@ [ ( "test_single_stream_slicer", - [ListStreamSlicer(["customer", "store", "subscription"], "owner_resource", None)], + [ListStreamSlicer(slice_values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, options={})], [{"owner_resource": "customer"}, {"owner_resource": "store"}, {"owner_resource": "subscription"}], ), ( "test_two_stream_slicers", [ - ListStreamSlicer(["customer", "store", "subscription"], "owner_resource", None), - ListStreamSlicer(["A", "B"], "letter", None), + ListStreamSlicer(slice_values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, options={}), + ListStreamSlicer(slice_values=["A", "B"], cursor_field="letter", config={}, options={}), ], [ {"owner_resource": "customer", "letter": "A"}, @@ -38,14 +38,15 @@ ( "test_list_and_datetime", [ - ListStreamSlicer(["customer", "store", "subscription"], "owner_resource", None), + ListStreamSlicer(slice_values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, options={}), DatetimeStreamSlicer( - MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d"), - MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d"), - "1d", - InterpolatedString.create("", options={}), - "%Y-%m-%d", - None, + start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", options={}), + end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", options={}), + step="1d", + cursor_field=InterpolatedString.create("", options={}), + datetime_format="%Y-%m-%d", + config={}, + options={}, ), ], [ @@ -63,7 +64,7 @@ ], ) def test_substream_slicer(test_name, stream_slicers, expected_slices): - slicer = CartesianProductStreamSlicer(stream_slicers) + slicer = CartesianProductStreamSlicer(stream_slicers=stream_slicers, options={}) slices = [s for s in slicer.stream_slices(SyncMode.incremental, stream_state=None)] assert slices == expected_slices @@ -82,17 +83,18 @@ def test_substream_slicer(test_name, stream_slicers, expected_slices): ) def test_update_cursor(test_name, stream_slice, expected_state): stream_slicers = [ - ListStreamSlicer(["customer", "store", "subscription"], "owner_resource", None), + ListStreamSlicer(slice_values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, options={}), DatetimeStreamSlicer( - MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d"), - MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d"), - "1d", - InterpolatedString("date"), - "%Y-%m-%d", - None, + start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", options={}), + end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", options={}), + step="1d", + cursor_field=InterpolatedString(string="date", options={}), + datetime_format="%Y-%m-%d", + config={}, + options={}, ), ] - slicer = CartesianProductStreamSlicer(stream_slicers) + slicer = CartesianProductStreamSlicer(stream_slicers=stream_slicers, options={}) slicer.update_cursor(stream_slice, None) updated_state = slicer.get_stream_state() assert expected_state == updated_state @@ -103,8 +105,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): [ ( "test_param_header", - RequestOption(RequestOptionType.request_parameter, "owner"), - RequestOption(RequestOptionType.header, "repo"), + RequestOption(inject_into=RequestOptionType.request_parameter, options={}, field_name="owner"), + RequestOption(inject_into=RequestOptionType.header, options={}, field_name="repo"), {"owner": "customer"}, {"repo": "airbyte"}, {}, @@ -112,8 +114,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ), ( "test_header_header", - RequestOption(RequestOptionType.header, "owner"), - RequestOption(RequestOptionType.header, "repo"), + RequestOption(inject_into=RequestOptionType.header, options={}, field_name="owner"), + RequestOption(inject_into=RequestOptionType.header, options={}, field_name="repo"), {}, {"owner": "customer", "repo": "airbyte"}, {}, @@ -121,8 +123,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ), ( "test_body_data", - RequestOption(RequestOptionType.body_data, "owner"), - RequestOption(RequestOptionType.body_data, "repo"), + RequestOption(inject_into=RequestOptionType.body_data, options={}, field_name="owner"), + RequestOption(inject_into=RequestOptionType.body_data, options={}, field_name="repo"), {}, {}, {}, @@ -130,8 +132,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ), ( "test_body_json", - RequestOption(RequestOptionType.body_json, "owner"), - RequestOption(RequestOptionType.body_json, "repo"), + RequestOption(inject_into=RequestOptionType.body_json, options={}, field_name="owner"), + RequestOption(inject_into=RequestOptionType.body_json, options={}, field_name="repo"), {}, {}, {"owner": "customer", "repo": "airbyte"}, @@ -149,14 +151,27 @@ def test_request_option( expected_body_data, ): slicer = CartesianProductStreamSlicer( - [ - ListStreamSlicer(["customer", "store", "subscription"], "owner_resource", None, stream_1_request_option), - ListStreamSlicer(["airbyte", "airbyte-cloud"], "repository", None, stream_2_request_option), - ] + stream_slicers=[ + ListStreamSlicer( + slice_values=["customer", "store", "subscription"], + cursor_field="owner_resource", + config={}, + request_option=stream_1_request_option, + options={}, + ), + ListStreamSlicer( + slice_values=["airbyte", "airbyte-cloud"], + cursor_field="repository", + config={}, + request_option=stream_2_request_option, + options={}, + ), + ], + options={}, ) slicer.update_cursor({"owner_resource": "customer", "repository": "airbyte"}, None) - assert expected_req_params == slicer.request_params() - assert expected_headers == slicer.request_headers() - assert expected_body_json == slicer.request_body_json() - assert expected_body_data == slicer.request_body_data() + assert expected_req_params == slicer.get_request_params() + assert expected_headers == slicer.get_request_headers() + assert expected_body_json == slicer.get_request_body_json() + assert expected_body_data == slicer.get_request_body_data() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py index a8c8cd287462..df1aa811ce11 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py @@ -16,9 +16,7 @@ FAKE_NOW = datetime.datetime(2022, 1, 1, tzinfo=datetime.timezone.utc) config = {"start_date": "2021-01-01T00:00:00.000000+0000", "start_date_ymd": "2021-01-01"} -end_date_now = InterpolatedString( - "{{ today_utc() }}", -) +end_date_now = InterpolatedString(string="{{ today_utc() }}", options={}) cursor_field = "created" timezone = datetime.timezone.utc @@ -36,8 +34,8 @@ def mock_datetime_now(monkeypatch): ( "test_1_day", None, - MinMaxDatetime("{{ config['start_date'] }}"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="{{ config['start_date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "1d", cursor_field, None, @@ -58,8 +56,8 @@ def mock_datetime_now(monkeypatch): ( "test_2_day", None, - MinMaxDatetime("{{ config['start_date'] }}"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="{{ config['start_date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "2d", cursor_field, None, @@ -75,8 +73,8 @@ def mock_datetime_now(monkeypatch): ( "test_from_stream_state", {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime("{{ stream_state['date'] }}"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="{{ stream_state['date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "1d", cursor_field, None, @@ -93,8 +91,8 @@ def mock_datetime_now(monkeypatch): ( "test_12_day", None, - MinMaxDatetime("{{ config['start_date'] }}"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="{{ config['start_date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "12d", cursor_field, None, @@ -106,8 +104,8 @@ def mock_datetime_now(monkeypatch): ( "test_end_time_greater_than_now", None, - MinMaxDatetime("2021-12-28T00:00:00.000000+0000"), - MinMaxDatetime(f"{(FAKE_NOW + datetime.timedelta(days=1)).strftime(datetime_format)}"), + MinMaxDatetime(datetime="2021-12-28T00:00:00.000000+0000", options={}), + MinMaxDatetime(datetime=f"{(FAKE_NOW + datetime.timedelta(days=1)).strftime(datetime_format)}", options={}), "1d", cursor_field, None, @@ -123,8 +121,8 @@ def mock_datetime_now(monkeypatch): ( "test_start_date_greater_than_end_time", None, - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), - MinMaxDatetime("2021-01-05T00:00:00.000000+0000"), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), + MinMaxDatetime(datetime="2021-01-05T00:00:00.000000+0000", options={}), "1d", cursor_field, None, @@ -136,10 +134,10 @@ def mock_datetime_now(monkeypatch): ( "test_cursor_date_greater_than_start_date", {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime("{{ stream_state['date'] }}"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="{{ stream_state['date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "1d", - InterpolatedString("{{ stream_state['date'] }}"), + InterpolatedString(string="{{ stream_state['date'] }}", options={}), None, datetime_format, [ @@ -154,8 +152,8 @@ def mock_datetime_now(monkeypatch): ( "test_cursor_date_greater_than_start_date_multiday_step", {cursor_field: "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime("2021-01-03T00:00:00.000000+0000"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="2021-01-03T00:00:00.000000+0000", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "2d", cursor_field, None, @@ -169,10 +167,10 @@ def mock_datetime_now(monkeypatch): ( "test_start_date_less_than_min_date", {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime("{{ config['start_date'] }}", min_datetime="{{ stream_state['date'] }}"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="{{ config['start_date'] }}", min_datetime="{{ stream_state['date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "1d", - InterpolatedString("{{ stream_state['date'] }}"), + InterpolatedString(string="{{ stream_state['date'] }}", options={}), None, datetime_format, [ @@ -187,8 +185,8 @@ def mock_datetime_now(monkeypatch): ( "test_end_date_greater_than_max_date", {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime("{{ config['start_date'] }}"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000", max_datetime="{{ stream_state['date'] }}"), + MinMaxDatetime(datetime="{{ config['start_date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", max_datetime="{{ stream_state['date'] }}", options={}), "1d", cursor_field, None, @@ -204,8 +202,8 @@ def mock_datetime_now(monkeypatch): ( "test_start_end_min_max_inherits_datetime_format_from_stream_slicer", {"date": "2021-01-05"}, - MinMaxDatetime("{{ config['start_date_ymd'] }}"), - MinMaxDatetime("2021-01-10", max_datetime="{{ stream_state['date'] }}"), + MinMaxDatetime(datetime="{{ config['start_date_ymd'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10", max_datetime="{{ stream_state['date'] }}", options={}), "1d", cursor_field, None, @@ -221,8 +219,8 @@ def mock_datetime_now(monkeypatch): ( "test_with_lookback_window_from_start_date", {"date": "2021-01-05"}, - MinMaxDatetime("{{ config['start_date'] }}"), - MinMaxDatetime("2021-01-10", max_datetime="{{ stream_state['date'] }}", datetime_format="%Y-%m-%d"), + MinMaxDatetime(datetime="{{ config['start_date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10", max_datetime="{{ stream_state['date'] }}", datetime_format="%Y-%m-%d", options={}), "1d", cursor_field, "3d", @@ -241,8 +239,8 @@ def mock_datetime_now(monkeypatch): ( "test_with_lookback_window_defaults_to_0d", {"date": "2021-01-05"}, - MinMaxDatetime("{{ config['start_date'] }}"), - MinMaxDatetime("2021-01-10", max_datetime="{{ stream_state['date'] }}", datetime_format="%Y-%m-%d"), + MinMaxDatetime(datetime="{{ config['start_date'] }}", options={}), + MinMaxDatetime(datetime="2021-01-10", max_datetime="{{ stream_state['date'] }}", datetime_format="%Y-%m-%d", options={}), "1d", cursor_field, "{{ config['does_not_exist'] }}", @@ -258,8 +256,8 @@ def mock_datetime_now(monkeypatch): ( "test_start_is_after_stream_state", {cursor_field: "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime("2021-01-01T00:00:00.000000+0000"), - MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", options={}), + MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), "1d", cursor_field, None, @@ -277,7 +275,7 @@ def mock_datetime_now(monkeypatch): def test_stream_slices( mock_datetime_now, test_name, stream_state, start, end, step, cursor_field, lookback_window, datetime_format, expected_slices ): - lookback_window = InterpolatedString(lookback_window) if lookback_window else None + lookback_window = InterpolatedString(string=lookback_window, options={}) if lookback_window else None slicer = DatetimeStreamSlicer( start_datetime=start, end_datetime=end, @@ -286,6 +284,7 @@ def test_stream_slices( datetime_format=datetime_format, lookback_window=lookback_window, config=config, + options={}, ) stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state) @@ -335,13 +334,14 @@ def test_stream_slices( ) def test_update_cursor(test_name, previous_cursor, stream_slice, last_record, expected_state): slicer = DatetimeStreamSlicer( - start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000"), - end_datetime=MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", options={}), + end_datetime=MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), step="1d", - cursor_field=InterpolatedString(cursor_field), + cursor_field=InterpolatedString(string=cursor_field, options={}), datetime_format=datetime_format, - lookback_window=InterpolatedString("0d"), + lookback_window=InterpolatedString(string="0d", options={}), config=config, + options={}, ) slicer._cursor = previous_cursor slicer.update_cursor(stream_slice, last_record) @@ -402,45 +402,47 @@ def test_update_cursor(test_name, previous_cursor, stream_slice, last_record, ex ) def test_request_option(test_name, inject_into, field_name, expected_req_params, expected_headers, expected_body_json, expected_body_data): if inject_into == RequestOptionType.path: - start_request_option = RequestOption(inject_into) - end_request_option = RequestOption(inject_into) + start_request_option = RequestOption(inject_into=inject_into, options={}) + end_request_option = RequestOption(inject_into=inject_into, options={}) try: DatetimeStreamSlicer( - start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000"), - end_datetime=MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", options={}), + end_datetime=MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), step="1d", - cursor_field=InterpolatedString(cursor_field), + cursor_field=InterpolatedString(string=cursor_field, options={}), datetime_format=datetime_format, - lookback_window=InterpolatedString("0d"), + lookback_window=InterpolatedString(string="0d", options={}), start_time_option=start_request_option, end_time_option=end_request_option, config=config, + options={}, ) assert False except ValueError: return else: - start_request_option = RequestOption(inject_into, field_name) if inject_into else None - end_request_option = RequestOption(inject_into, "endtime") if inject_into else None + start_request_option = RequestOption(inject_into=inject_into, options={}, field_name=field_name) if inject_into else None + end_request_option = RequestOption(inject_into=inject_into, options={}, field_name="endtime") if inject_into else None slicer = DatetimeStreamSlicer( - start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000"), - end_datetime=MinMaxDatetime("2021-01-10T00:00:00.000000+0000"), + start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", options={}), + end_datetime=MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", options={}), step="1d", - cursor_field=InterpolatedString(cursor_field), + cursor_field=InterpolatedString(string=cursor_field, options={}), datetime_format=datetime_format, - lookback_window=InterpolatedString("0d"), + lookback_window=InterpolatedString(string="0d", options={}), start_time_option=start_request_option, end_time_option=end_request_option, config=config, + options={}, ) stream_slice = {"start_time": "2021-01-01T00:00:00.000000+0000", "end_time": "2021-01-04T00:00:00.000000+0000"} slicer.update_cursor(stream_slice) - assert expected_req_params == slicer.request_params(stream_slice=stream_slice) - assert expected_headers == slicer.request_headers(stream_slice=stream_slice) - assert expected_body_json == slicer.request_body_json(stream_slice=stream_slice) - assert expected_body_data == slicer.request_body_data(stream_slice=stream_slice) + assert expected_req_params == slicer.get_request_params(stream_slice=stream_slice) + assert expected_headers == slicer.get_request_headers(stream_slice=stream_slice) + assert expected_body_json == slicer.get_request_body_json(stream_slice=stream_slice) + assert expected_body_data == slicer.get_request_body_data(stream_slice=stream_slice) if __name__ == "__main__": diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_stream_slicer.py similarity index 61% rename from airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_slicer.py rename to airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_stream_slicer.py index ccb8ef40803c..1245a7c14ba0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_stream_slicer.py @@ -9,6 +9,7 @@ slice_values = ["customer", "store", "subscription"] cursor_field = "owner_resource" +options = {"cursor_field": "owner_resource"} @pytest.mark.parametrize( @@ -26,10 +27,16 @@ "owner_resource", [{"owner_resource": "customer"}, {"owner_resource": "store"}, {"owner_resource": "subscription"}], ), + ( + "test_using_cursor_from_options", + '["customer", "store", "subscription"]', + "{{ options['cursor_field'] }}", + [{"owner_resource": "customer"}, {"owner_resource": "store"}, {"owner_resource": "subscription"}], + ), ], ) -def test_list_slicer(test_name, slice_values, cursor_field, expected_slices): - slicer = ListStreamSlicer(slice_values, cursor_field, config={}) +def test_list_stream_slicer(test_name, slice_values, cursor_field, expected_slices): + slicer = ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, options=options) slices = [s for s in slicer.stream_slices(SyncMode.incremental, stream_state=None)] assert slices == expected_slices @@ -43,7 +50,7 @@ def test_list_slicer(test_name, slice_values, cursor_field, expected_slices): ], ) def test_update_cursor(test_name, stream_slice, last_record, expected_state): - slicer = ListStreamSlicer(slice_values, cursor_field, config={}) + slicer = ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, options={}) slicer.update_cursor(stream_slice, last_record) updated_state = slicer.get_stream_state() assert expected_state == updated_state @@ -54,16 +61,23 @@ def test_update_cursor(test_name, stream_slice, last_record, expected_state): [ ( "test_inject_into_req_param", - RequestOption(RequestOptionType.request_parameter, "owner_resource"), + RequestOption(inject_into=RequestOptionType.request_parameter, options={}, field_name="owner_resource"), {"owner_resource": "customer"}, {}, {}, {}, ), - ("test_pass_by_header", RequestOption(RequestOptionType.header, "owner_resource"), {}, {"owner_resource": "customer"}, {}, {}), + ( + "test_pass_by_header", + RequestOption(inject_into=RequestOptionType.header, options={}, field_name="owner_resource"), + {}, + {"owner_resource": "customer"}, + {}, + {}, + ), ( "test_inject_into_body_json", - RequestOption(RequestOptionType.body_json, "owner_resource"), + RequestOption(inject_into=RequestOptionType.body_json, options={}, field_name="owner_resource"), {}, {}, {"owner_resource": "customer"}, @@ -71,7 +85,7 @@ def test_update_cursor(test_name, stream_slice, last_record, expected_state): ), ( "test_inject_into_body_data", - RequestOption(RequestOptionType.body_data, "owner_resource"), + RequestOption(inject_into=RequestOptionType.body_data, options={}, field_name="owner_resource"), {}, {}, {}, @@ -79,7 +93,7 @@ def test_update_cursor(test_name, stream_slice, last_record, expected_state): ), ( "test_inject_into_path", - RequestOption(RequestOptionType.path), + RequestOption(RequestOptionType.path, {}), {}, {}, {}, @@ -90,15 +104,15 @@ def test_update_cursor(test_name, stream_slice, last_record, expected_state): def test_request_option(test_name, request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data): if request_option.inject_into == RequestOptionType.path: try: - ListStreamSlicer(slice_values, cursor_field, {}, request_option) + ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, request_option=request_option, options={}) assert False except ValueError: return - slicer = ListStreamSlicer(slice_values, cursor_field, {}, request_option) + slicer = ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, request_option=request_option, options={}) stream_slice = {cursor_field: "customer"} slicer.update_cursor(stream_slice) - assert expected_req_params == slicer.request_params(stream_slice) - assert expected_headers == slicer.request_headers() - assert expected_body_json == slicer.request_body_json() - assert expected_body_data == slicer.request_body_data() + assert expected_req_params == slicer.get_request_params(stream_slice) + assert expected_headers == slicer.get_request_headers() + assert expected_body_json == slicer.get_request_body_json() + assert expected_body_data == slicer.get_request_body_data() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py index d3e15a5c9fba..d4c8d5ad1f1f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py @@ -59,17 +59,35 @@ def read_records( ("test_no_parents", [], None), ( "test_single_parent_slices_no_records", - [ParentStreamConfig(MockStream([{}], [], "first_stream"), "id", "first_stream_id")], + [ + ParentStreamConfig( + stream=MockStream([{}], [], "first_stream"), parent_key="id", stream_slice_field="first_stream_id", options={} + ) + ], [{"first_stream_id": None, "parent_slice": None}], ), ( "test_single_parent_slices_with_records", - [ParentStreamConfig(MockStream([{}], parent_records, "first_stream"), "id", "first_stream_id")], + [ + ParentStreamConfig( + stream=MockStream([{}], parent_records, "first_stream"), + parent_key="id", + stream_slice_field="first_stream_id", + options={}, + ) + ], [{"first_stream_id": 1, "parent_slice": None}, {"first_stream_id": 2, "parent_slice": None}], ), ( "test_with_parent_slices_and_records", - [ParentStreamConfig(MockStream(parent_slices, all_parent_data, "first_stream"), "id", "first_stream_id")], + [ + ParentStreamConfig( + stream=MockStream(parent_slices, all_parent_data, "first_stream"), + parent_key="id", + stream_slice_field="first_stream_id", + options={}, + ) + ], [ {"parent_slice": "first", "first_stream_id": 0}, {"parent_slice": "first", "first_stream_id": 1}, @@ -81,9 +99,17 @@ def read_records( "test_multiple_parent_streams", [ ParentStreamConfig( - MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), "id", "first_stream_id" + stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), + parent_key="id", + stream_slice_field="first_stream_id", + options={}, + ), + ParentStreamConfig( + stream=MockStream(second_parent_stream_slice, more_records, "second_stream"), + parent_key="id", + stream_slice_field="second_stream_id", + options={}, ), - ParentStreamConfig(MockStream(second_parent_stream_slice, more_records, "second_stream"), "id", "second_stream_id"), ], [ {"parent_slice": "first", "first_stream_id": 0}, @@ -99,11 +125,11 @@ def read_records( def test_substream_slicer(test_name, parent_stream_configs, expected_slices): if expected_slices is None: try: - SubstreamSlicer(parent_stream_configs) + SubstreamSlicer(parent_stream_configs=parent_stream_configs, options={}) assert False except ValueError: return - slicer = SubstreamSlicer(parent_stream_configs) + slicer = SubstreamSlicer(parent_stream_configs=parent_stream_configs, options={}) slices = [s for s in slicer.stream_slices(SyncMode.incremental, stream_state=None)] assert slices == expected_slices @@ -124,12 +150,20 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): def test_update_cursor(test_name, stream_slice, expected_state): parent_stream_name_to_config = [ ParentStreamConfig( - MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), "id", "first_stream_id" + stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), + parent_key="id", + stream_slice_field="first_stream_id", + options={}, + ), + ParentStreamConfig( + stream=MockStream(second_parent_stream_slice, more_records, "second_stream"), + parent_key="id", + stream_slice_field="second_stream_id", + options={}, ), - ParentStreamConfig(MockStream(second_parent_stream_slice, more_records, "second_stream"), "id", "second_stream_id"), ] - slicer = SubstreamSlicer(parent_stream_name_to_config) + slicer = SubstreamSlicer(parent_stream_configs=parent_stream_name_to_config, options={}) slicer.update_cursor(stream_slice, None) updated_state = slicer.get_stream_state() assert expected_state == updated_state @@ -141,8 +175,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ( "test_request_option_in_request_param", [ - RequestOption(RequestOptionType.request_parameter, "first_stream"), - RequestOption(RequestOptionType.request_parameter, "second_stream"), + RequestOption(inject_into=RequestOptionType.request_parameter, options={}, field_name="first_stream"), + RequestOption(inject_into=RequestOptionType.request_parameter, options={}, field_name="second_stream"), ], {"first_stream_id": "1234", "second_stream_id": "4567"}, {}, @@ -152,8 +186,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ( "test_request_option_in_header", [ - RequestOption(RequestOptionType.header, "first_stream"), - RequestOption(RequestOptionType.header, "second_stream"), + RequestOption(inject_into=RequestOptionType.header, options={}, field_name="first_stream"), + RequestOption(inject_into=RequestOptionType.header, options={}, field_name="second_stream"), ], {}, {"first_stream_id": "1234", "second_stream_id": "4567"}, @@ -163,8 +197,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ( "test_request_option_in_param_and_header", [ - RequestOption(RequestOptionType.request_parameter, "first_stream"), - RequestOption(RequestOptionType.header, "second_stream"), + RequestOption(inject_into=RequestOptionType.request_parameter, options={}, field_name="first_stream"), + RequestOption(inject_into=RequestOptionType.header, options={}, field_name="second_stream"), ], {"first_stream_id": "1234"}, {"second_stream_id": "4567"}, @@ -174,8 +208,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ( "test_request_option_in_body_json", [ - RequestOption(RequestOptionType.body_json, "first_stream"), - RequestOption(RequestOptionType.body_json, "second_stream"), + RequestOption(inject_into=RequestOptionType.body_json, options={}, field_name="first_stream"), + RequestOption(inject_into=RequestOptionType.body_json, options={}, field_name="second_stream"), ], {}, {}, @@ -185,8 +219,8 @@ def test_update_cursor(test_name, stream_slice, expected_state): ( "test_request_option_in_body_data", [ - RequestOption(RequestOptionType.body_data, "first_stream"), - RequestOption(RequestOptionType.body_data, "second_stream"), + RequestOption(inject_into=RequestOptionType.body_data, options={}, field_name="first_stream"), + RequestOption(inject_into=RequestOptionType.body_data, options={}, field_name="second_stream"), ], {}, {}, @@ -204,24 +238,27 @@ def test_request_option( expected_body_data, ): slicer = SubstreamSlicer( - [ + parent_stream_configs=[ ParentStreamConfig( - MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), - "id", - "first_stream_id", - parent_stream_request_options[0], + stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), + parent_key="id", + stream_slice_field="first_stream_id", + options={}, + request_option=parent_stream_request_options[0], ), ParentStreamConfig( - MockStream(second_parent_stream_slice, more_records, "second_stream"), - "id", - "second_stream_id", - parent_stream_request_options[1], + stream=MockStream(second_parent_stream_slice, more_records, "second_stream"), + parent_key="id", + stream_slice_field="second_stream_id", + options={}, + request_option=parent_stream_request_options[1], ), ], + options={}, ) slicer.update_cursor({"first_stream_id": "1234", "second_stream_id": "4567"}, None) - assert expected_req_params == slicer.request_params() - assert expected_headers == slicer.request_headers() - assert expected_body_json == slicer.request_body_json() - assert expected_body_data == slicer.request_body_data() + assert expected_req_params == slicer.get_request_params() + assert expected_headers == slicer.get_request_headers() + assert expected_body_json == slicer.get_request_body_json() + assert expected_body_data == slicer.get_request_body_data() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py index cb239d0eca17..3ba79ab81e7d 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py @@ -7,9 +7,10 @@ class AClass: - def __init__(self, parameter, another_param): + def __init__(self, parameter, another_param, options): self.parameter = parameter self.another_param = another_param + self.options = options class OuterClass: @@ -42,12 +43,20 @@ def test_string_interpolation(): s = "{{ next_page_token['next_page_url'] }}" partial = create(InterpolatedString, string=s) interpolated_string = partial() - assert interpolated_string._string == s + assert interpolated_string.string == s def test_string_interpolation_through_kwargs(): s = "{{ options['name'] }}" options = {"name": "airbyte"} - partial = create(InterpolatedString, string=s, options=options) + partial = create(InterpolatedString, string=s, **options) + interpolated_string = partial() + assert interpolated_string.eval({}) == "airbyte" + + +def test_string_interpolation_through_options_keyword(): + s = "{{ options['name'] }}" + options = {"$options": {"name": "airbyte"}} + partial = create(InterpolatedString, string=s, **options) interpolated_string = partial() assert interpolated_string.eval({}) == "airbyte" diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py index 55295e7ca377..1b6b84906271 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py @@ -38,12 +38,13 @@ def test_declarative_stream(): stream = DeclarativeStream( name=name, primary_key=primary_key, - cursor_field=cursor_field, + stream_cursor_field=cursor_field, schema_loader=schema_loader, retriever=retriever, config=config, transformations=transformations, checkpoint_interval=checkpoint_interval, + options={}, ) assert stream.name == name diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index 20a2cf248812..190c44846047 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector +from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.parsers.factory import DeclarativeComponentFactory from airbyte_cdk.sources.declarative.parsers.yaml_parser import YamlParser from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import CompositeErrorHandler @@ -44,6 +45,8 @@ def test_factory(): offset: "{{ next_page_token['offset'] }}" limit: "*ref(limit)" request_options: + $options: + here: "iam" class_name: airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider.InterpolatedRequestOptionsProvider request_parameters: "*ref(offset_request_parameters)" request_body_json: @@ -51,11 +54,12 @@ def test_factory(): """ config = parser.parse(content) request_options_provider = factory.create_component(config["request_options"], input_config)() + assert type(request_options_provider) == InterpolatedRequestOptionsProvider assert request_options_provider._parameter_interpolator._config == input_config - assert request_options_provider._parameter_interpolator._interpolator._mapping["offset"] == "{{ next_page_token['offset'] }}" + assert request_options_provider._parameter_interpolator._interpolator.mapping["offset"] == "{{ next_page_token['offset'] }}" assert request_options_provider._body_json_interpolator._config == input_config - assert request_options_provider._body_json_interpolator._interpolator._mapping["body_offset"] == "{{ next_page_token['offset'] }}" + assert request_options_provider._body_json_interpolator._interpolator.mapping["body_offset"] == "{{ next_page_token['offset'] }}" def test_interpolate_config(): @@ -72,12 +76,13 @@ def test_interpolate_config(): """ config = parser.parse(content) authenticator = factory.create_component(config["authenticator"], input_config)() - assert authenticator._client_id.eval(input_config) == "some_client_id" - assert authenticator._client_secret._string == "some_client_secret" + assert authenticator.client_id.eval(input_config) == "some_client_id" + assert authenticator.client_secret.string == "some_client_secret" - assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" - assert authenticator._refresh_token.eval(input_config) == "verysecrettoken" - assert authenticator._refresh_request_body._mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"} + assert authenticator.token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" + assert authenticator.refresh_token.eval(input_config) == "verysecrettoken" + assert authenticator._refresh_request_body.mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"} + assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"} def test_list_based_stream_slicer_with_values_refd(): @@ -90,7 +95,7 @@ def test_list_based_stream_slicer_with_values_refd(): """ config = parser.parse(content) stream_slicer = factory.create_component(config["stream_slicer"], input_config)() - assert ["airbyte", "airbyte-cloud"] == stream_slicer._slice_values + assert ["airbyte", "airbyte-cloud"] == stream_slicer.slice_values def test_list_based_stream_slicer_with_values_defined_in_config(): @@ -105,17 +110,19 @@ def test_list_based_stream_slicer_with_values_defined_in_config(): """ config = parser.parse(content) stream_slicer = factory.create_component(config["stream_slicer"], input_config)() - assert ["airbyte", "airbyte-cloud"] == stream_slicer._slice_values - assert stream_slicer._request_option._option_type == RequestOptionType.header - assert stream_slicer._request_option._field_name == "repository" + assert ["airbyte", "airbyte-cloud"] == stream_slicer.slice_values + assert stream_slicer.request_option.inject_into == RequestOptionType.header + assert stream_slicer.request_option.field_name == "repository" def test_create_substream_slicer(): content = """ schema_loader: - file_path: "./source_sendgrid/schemas/{{name}}.yaml" + file_path: "./source_sendgrid/schemas/{{ options['stream_name'] }}.yaml" + name: "{{ options['stream_name'] }}" retriever: requester: + name: "{{ options['stream_name'] }}" path: "/v3" record_selector: extractor: @@ -123,22 +130,22 @@ def test_create_substream_slicer(): stream_A: type: DeclarativeStream $options: - name: "A" - primary_key: "id" + stream_name: "A" + stream_primary_key: "id" retriever: "*ref(retriever)" url_base: "https://airbyte.io" schema_loader: "*ref(schema_loader)" stream_B: type: DeclarativeStream $options: - name: "B" - primary_key: "id" + stream_name: "B" + stream_primary_key: "id" retriever: "*ref(retriever)" url_base: "https://airbyte.io" schema_loader: "*ref(schema_loader)" stream_slicer: type: SubstreamSlicer - parent_streams_configs: + parent_stream_configs: - stream: "*ref(stream_A)" parent_key: id stream_slice_field: repository_id @@ -151,18 +158,18 @@ def test_create_substream_slicer(): """ config = parser.parse(content) stream_slicer = factory.create_component(config["stream_slicer"], input_config)() - parent_stream_configs = stream_slicer._parent_stream_configs + parent_stream_configs = stream_slicer.parent_stream_configs assert len(parent_stream_configs) == 2 assert isinstance(parent_stream_configs[0].stream, DeclarativeStream) assert isinstance(parent_stream_configs[1].stream, DeclarativeStream) - assert stream_slicer._parent_stream_configs[0].parent_key == "id" - assert stream_slicer._parent_stream_configs[0].stream_slice_field == "repository_id" - assert stream_slicer._parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter - assert stream_slicer._parent_stream_configs[0].request_option._field_name == "repository_id" + assert stream_slicer.parent_stream_configs[0].parent_key == "id" + assert stream_slicer.parent_stream_configs[0].stream_slice_field == "repository_id" + assert stream_slicer.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter + assert stream_slicer.parent_stream_configs[0].request_option.field_name == "repository_id" - assert stream_slicer._parent_stream_configs[1].parent_key == "someid" - assert stream_slicer._parent_stream_configs[1].stream_slice_field == "word_id" - assert stream_slicer._parent_stream_configs[1].request_option is None + assert stream_slicer.parent_stream_configs[1].parent_key == "someid" + assert stream_slicer.parent_stream_configs[1].stream_slice_field == "word_id" + assert stream_slicer.parent_stream_configs[1].request_option is None def test_create_cartesian_stream_slicer(): @@ -185,12 +192,12 @@ def test_create_cartesian_stream_slicer(): """ config = parser.parse(content) stream_slicer = factory.create_component(config["stream_slicer"], input_config)() - underlying_slicers = stream_slicer._stream_slicers + underlying_slicers = stream_slicer.stream_slicers assert len(underlying_slicers) == 2 assert isinstance(underlying_slicers[0], ListStreamSlicer) assert isinstance(underlying_slicers[1], ListStreamSlicer) - assert ["airbyte", "airbyte-cloud"] == underlying_slicers[0]._slice_values - assert ["hello", "world"] == underlying_slicers[1]._slice_values + assert ["airbyte", "airbyte-cloud"] == underlying_slicers[0].slice_values + assert ["hello", "world"] == underlying_slicers[1].slice_values def test_datetime_stream_slicer(): @@ -216,18 +223,18 @@ def test_datetime_stream_slicer(): stream_slicer = factory.create_component(config["stream_slicer"], input_config)() assert type(stream_slicer) == DatetimeStreamSlicer assert stream_slicer._timezone == datetime.timezone.utc - assert type(stream_slicer._start_datetime) == MinMaxDatetime - assert type(stream_slicer._end_datetime) == MinMaxDatetime - assert stream_slicer._start_datetime._datetime_format == "%Y-%m-%dT%H:%M:%S.%f%z" - assert stream_slicer._start_datetime._timezone == datetime.timezone.utc - assert stream_slicer._start_datetime._datetime_interpolator._string == "{{ config['start_time'] }}" - assert stream_slicer._start_datetime._min_datetime_interpolator._string == "{{ config['start_time'] + day_delta(2) }}" - assert stream_slicer._end_datetime._datetime_interpolator._string == "{{ config['end_time'] }}" + assert type(stream_slicer.start_datetime) == MinMaxDatetime + assert type(stream_slicer.end_datetime) == MinMaxDatetime + assert stream_slicer.start_datetime._datetime_format == "%Y-%m-%dT%H:%M:%S.%f%z" + assert stream_slicer.start_datetime._timezone == datetime.timezone.utc + assert stream_slicer.start_datetime.datetime.string == "{{ config['start_time'] }}" + assert stream_slicer.start_datetime.min_datetime.string == "{{ config['start_time'] + day_delta(2) }}" + assert stream_slicer.end_datetime.datetime.string == "{{ config['end_time'] }}" assert stream_slicer._step == datetime.timedelta(days=10) - assert stream_slicer._cursor_field._string == "created" - assert stream_slicer._lookback_window._string == "5d" - assert stream_slicer._start_time_option.inject_into == RequestOptionType.request_parameter - assert stream_slicer._start_time_option._field_name == "created[gte]" + assert stream_slicer.cursor_field.string == "created" + assert stream_slicer.lookback_window.string == "5d" + assert stream_slicer.start_time_option.inject_into == RequestOptionType.request_parameter + assert stream_slicer.start_time_option.field_name == "created[gte]" def test_full_config(): @@ -266,7 +273,7 @@ def test_full_config(): http_method: "GET" authenticator: type: BearerAuthenticator - token: "{{ config['apikey'] }}" + api_token: "{{ config['apikey'] }}" request_parameters_provider: "*ref(request_options_provider)" error_handler: type: DefaultErrorHandler @@ -314,29 +321,29 @@ def test_full_config(): assert stream_config["cursor_field"] == [] stream = factory.create_component(stream_config, input_config)() - assert isinstance(stream._retriever._record_selector._extractor, JelloExtractor) + assert isinstance(stream.retriever.record_selector.extractor, JelloExtractor) assert type(stream) == DeclarativeStream assert stream.primary_key == "id" assert stream.name == "lists" - assert type(stream._schema_loader) == JsonSchema - assert type(stream._retriever) == SimpleRetriever - assert stream._retriever._requester._method == HttpMethod.GET - assert stream._retriever._requester._authenticator._token.eval(input_config) == "verysecrettoken" - assert type(stream._retriever._record_selector) == RecordSelector - assert type(stream._retriever._record_selector._extractor._decoder) == JsonDecoder - - assert stream._retriever._record_selector._extractor._transform.eval(input_config) == "_.result" - assert type(stream._retriever._record_selector._record_filter) == RecordFilter - assert stream._retriever._record_selector._record_filter._filter_interpolator._condition == "{{ record['id'] > stream_state['id'] }}" - assert stream._schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json" + assert type(stream.schema_loader) == JsonSchema + assert type(stream.retriever) == SimpleRetriever + assert stream.retriever.requester.http_method == HttpMethod.GET + assert stream.retriever.requester.authenticator._token.eval(input_config) == "verysecrettoken" + assert type(stream.retriever.record_selector) == RecordSelector + assert type(stream.retriever.record_selector.extractor.decoder) == JsonDecoder + + assert stream.retriever.record_selector.extractor.transform.eval(input_config) == "_.result" + assert type(stream.retriever.record_selector.record_filter) == RecordFilter + assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}" + assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json" checker = factory.create_component(config["check"], input_config)() - streams_to_check = checker._stream_names + streams_to_check = checker.stream_names assert len(streams_to_check) == 1 assert list(streams_to_check)[0] == "list_stream" - assert stream._retriever._requester._path._default == "marketing/lists" + assert stream.retriever.requester.path.default == "marketing/lists" def test_create_record_selector(): @@ -356,9 +363,9 @@ def test_create_record_selector(): config = parser.parse(content) selector = factory.create_component(config["selector"], input_config)() assert isinstance(selector, RecordSelector) - assert isinstance(selector._extractor, JelloExtractor) - assert selector._extractor._transform.eval(input_config) == "_.result" - assert isinstance(selector._record_filter, RecordFilter) + assert isinstance(selector.extractor, JelloExtractor) + assert selector.extractor.transform.eval(input_config) == "_.result" + assert isinstance(selector.record_filter, RecordFilter) def test_create_requester(): @@ -367,7 +374,7 @@ def test_create_requester(): type: HttpRequester path: "/v3/marketing/lists" $options: - name: lists + name: 'lists' url_base: "https://api.sendgrid.com" authenticator: type: "BasicHttpAuthenticator" @@ -382,16 +389,16 @@ def test_create_requester(): config = parser.parse(content) component = factory.create_component(config["requester"], input_config)() assert isinstance(component, HttpRequester) - assert isinstance(component._error_handler, DefaultErrorHandler) - assert component._path._string == "/v3/marketing/lists" - assert component._url_base._string == "https://api.sendgrid.com" - assert isinstance(component._authenticator, BasicHttpAuthenticator) - assert component._authenticator._username.eval(input_config) == "lists" - assert component._authenticator._password.eval(input_config) == "verysecrettoken" + assert isinstance(component.error_handler, DefaultErrorHandler) + assert component.path.string == "/v3/marketing/lists" + assert component.url_base.string == "https://api.sendgrid.com" + assert isinstance(component.authenticator, BasicHttpAuthenticator) + assert component.authenticator._username.eval(input_config) == "lists" + assert component.authenticator._password.eval(input_config) == "verysecrettoken" assert component._method == HttpMethod.GET - assert component._request_options_provider._parameter_interpolator._interpolator._mapping["page_size"] == 10 - assert component._request_options_provider._headers_interpolator._interpolator._mapping["header"] == "header_value" - assert component._name == "lists" + assert component._request_options_provider._parameter_interpolator._interpolator.mapping["page_size"] == 10 + assert component._request_options_provider._headers_interpolator._interpolator.mapping["header"] == "header_value" + assert component.name == "lists" def test_create_composite_error_handler(): @@ -408,11 +415,11 @@ def test_create_composite_error_handler(): """ config = parser.parse(content) component = factory.create_component(config["error_handler"], input_config)() - assert len(component._error_handlers) == 2 - assert isinstance(component._error_handlers[0], DefaultErrorHandler) - assert isinstance(component._error_handlers[0]._response_filters[0], HttpResponseFilter) - assert component._error_handlers[0]._response_filters[0]._predicate._condition == "{{ 'code' in response }}" - assert component._error_handlers[1]._response_filters[0]._http_codes == [403] + assert len(component.error_handlers) == 2 + assert isinstance(component.error_handlers[0], DefaultErrorHandler) + assert isinstance(component.error_handlers[0].response_filters[0], HttpResponseFilter) + assert component.error_handlers[0].response_filters[0].predicate.condition == "{{ 'code' in response }}" + assert component.error_handlers[1].response_filters[0].http_codes == [403] assert isinstance(component, CompositeErrorHandler) @@ -425,7 +432,8 @@ def test_config_with_defaults(): primary_key: id url_base: "https://api.sendgrid.com" schema_loader: - file_path: "./source_sendgrid/schemas/{{options.name}}.yaml" + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" retriever: paginator: type: "LimitPaginator" @@ -442,7 +450,7 @@ def test_config_with_defaults(): path: "/v3/marketing/lists" authenticator: type: "BearerAuthenticator" - token: "{{ config.apikey }}" + api_token: "{{ config.apikey }}" request_parameters: page_size: 10 record_selector: @@ -458,17 +466,17 @@ def test_config_with_defaults(): assert type(stream) == DeclarativeStream assert stream.primary_key == "id" assert stream.name == "lists" - assert type(stream._schema_loader) == JsonSchema - assert type(stream._retriever) == SimpleRetriever - assert stream._retriever._requester._method == HttpMethod.GET + assert type(stream.schema_loader) == JsonSchema + assert type(stream.retriever) == SimpleRetriever + assert stream.retriever.requester.http_method == HttpMethod.GET - assert stream._retriever._requester._authenticator._token.eval(input_config) == "verysecrettoken" - assert stream._retriever._record_selector._extractor._transform.eval(input_config) == "_.result" - assert stream._schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.yaml" - assert isinstance(stream._retriever._paginator, LimitPaginator) + assert stream.retriever.requester.authenticator._token.eval(input_config) == "verysecrettoken" + assert stream.retriever.record_selector.extractor.transform.eval(input_config) == "_.result" + assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.yaml" + assert isinstance(stream.retriever.paginator, LimitPaginator) - assert stream._retriever._paginator._url_base._string == "https://api.sendgrid.com" - assert stream._retriever._paginator._page_size == 10 + assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com" + assert stream.retriever.paginator.page_size == 10 def test_create_limit_paginator(): @@ -491,7 +499,7 @@ def test_create_limit_paginator(): paginator_config = config["paginator"] paginator = factory.create_component(paginator_config, input_config)() assert isinstance(paginator, LimitPaginator) - page_token_option = paginator._page_token_option + page_token_option = paginator.page_token_option assert isinstance(page_token_option, RequestOption) assert page_token_option.inject_into == RequestOptionType.path @@ -503,9 +511,11 @@ class TestCreateTransformations: primary_key: id url_base: "https://api.sendgrid.com" schema_loader: - file_path: "./source_sendgrid/schemas/{{name}}.yaml" + name: "{{ options.name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" retriever: requester: + name: "{{ options.name }}" path: "/v3/marketing/lists" request_parameters: page_size: 10 @@ -524,7 +534,7 @@ def test_no_transformations(self): config = parser.parse(content) component = factory.create_component(config["the_stream"], input_config)() assert isinstance(component, DeclarativeStream) - assert [] == component._transformations + assert [] == component.transformations def test_remove_fields(self): content = f""" @@ -541,8 +551,8 @@ def test_remove_fields(self): config = parser.parse(content) component = factory.create_component(config["the_stream"], input_config)() assert isinstance(component, DeclarativeStream) - expected = [RemoveFields(field_pointers=[["path", "to", "field1"], ["path2"]])] - assert expected == component._transformations + expected = [RemoveFields(field_pointers=[["path", "to", "field1"], ["path2"]], options={})] + assert expected == component.transformations def test_add_fields(self): content = f""" @@ -559,5 +569,14 @@ def test_add_fields(self): config = parser.parse(content) component = factory.create_component(config["the_stream"], input_config)() assert isinstance(component, DeclarativeStream) - expected = [AddFields([AddedFieldDefinition(["field1"], "static_value")])] - assert expected == component._transformations + expected = [ + AddFields( + fields=[ + AddedFieldDefinition( + path=["field1"], value=InterpolatedString(string="static_value", default="static_value", options={}), options={} + ) + ], + options={}, + ) + ] + assert expected == component.transformations diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py index 89941d4ba315..61fb31ba7056 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py @@ -106,5 +106,5 @@ def test_add_fields( input_record: Mapping[str, Any], field: List[Tuple[FieldPointer, str]], kwargs: Mapping[str, Any], expected: Mapping[str, Any] ): - inputs = [AddedFieldDefinition(v[0], v[1]) for v in field] - assert AddFields(inputs).transform(input_record, **kwargs) == expected + inputs = [AddedFieldDefinition(path=v[0], value=v[1], options={}) for v in field] + assert AddFields(fields=inputs, options={"alas": "i live"}).transform(input_record, **kwargs) == expected diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py index 2040794b26fb..c1d0358e4cdb 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_remove_fields.py @@ -44,5 +44,5 @@ ], ) def test_remove_fields(input_record: Mapping[str, Any], field_pointers: List[FieldPointer], expected: Mapping[str, Any]): - transformation = RemoveFields(field_pointers) + transformation = RemoveFields(field_pointers=field_pointers, options={}) assert transformation.transform(input_record) == expected diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index c70c88ecdbc4..36386d2143d6 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -128,7 +128,7 @@ def test_refresh_request_body(self): token_expiry_date=pendulum.now().add(days=3), refresh_request_body={"custom_field": "in_outbound_request", "another_field": "exists_in_body", "scopes": ["no_override"]}, ) - body = oauth.get_refresh_request_body() + body = oauth.build_refresh_request_body() expected = { "grant_type": "refresh_token", "client_id": "some_client_id", diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs index ad41adab2476..3ca79159242e 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs @@ -1,6 +1,6 @@ schema_loader: type: JsonSchema - file_path: "./source_{{snakeCase name}}/schemas/\{{ options.name }}.json" + file_path: "./source_{{snakeCase name}}/schemas/\{{ options['name'] }}.json" selector: type: RecordSelector extractor: @@ -9,13 +9,14 @@ selector: requester: type: HttpRequester name: "\{{ options['name'] }}" - url_base: TODO "your_api_base_url" http_method: "GET" authenticator: - type: TokenAuthenticator - token: "\{{ config['api_key'] }}" + type: ApiKeyAuthenticator + api_token: "\{{ config['api_key'] }}" retriever: type: SimpleRetriever + $options: + url_base: TODO "your_api_base_url" name: "\{{ options['name'] }}" primary_key: "\{{ options['primary_key'] }}" record_selector: From 5031e7242656f61be560f9f8917cae39b329bd3b Mon Sep 17 00:00:00 2001 From: Octavia Squidington III <90398440+octavia-squidington-iii@users.noreply.github.com> Date: Fri, 5 Aug 2022 14:43:31 -0700 Subject: [PATCH 23/25] Bump Airbyte version from 0.39.41-alpha to 0.39.42-alpha (#15335) Co-authored-by: xiaohansong Co-authored-by: Xiaohan Song --- .bumpversion.cfg | 2 +- .env | 2 +- airbyte-bootloader/Dockerfile | 2 +- airbyte-container-orchestrator/Dockerfile | 2 +- airbyte-metrics/reporter/Dockerfile | 2 +- airbyte-server/Dockerfile | 2 +- airbyte-webapp/package-lock.json | 4 ++-- airbyte-webapp/package.json | 2 +- airbyte-workers/Dockerfile | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.yaml | 2 +- charts/airbyte/README.md | 10 +++++----- charts/airbyte/values.yaml | 8 ++++---- docs/operator-guides/upgrading-airbyte.md | 2 +- kube/overlays/stable-with-resource-limits/.env | 2 +- .../stable-with-resource-limits/kustomization.yaml | 10 +++++----- kube/overlays/stable/.env | 2 +- kube/overlays/stable/kustomization.yaml | 10 +++++----- octavia-cli/Dockerfile | 2 +- octavia-cli/README.md | 2 +- octavia-cli/install.sh | 2 +- octavia-cli/setup.py | 2 +- 26 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c7938a2ca6a5..0f30162592de 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.39.41-alpha +current_version = 0.39.42-alpha commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index f7e7422357a7..2615b8753c61 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.39.41-alpha +VERSION=0.39.42-alpha # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index 0b090877291f..c90fa41524da 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye ARG JDK_IMAGE=openjdk:${JDK_VERSION} FROM ${JDK_IMAGE} -ARG VERSION=0.39.41-alpha +ARG VERSION=0.39.42-alpha ENV APPLICATION airbyte-bootloader ENV VERSION ${VERSION} diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index fcfe94867aff..e56a12bb04ab 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -28,7 +28,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] htt RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y kubectl # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.39.41-alpha +ARG VERSION=0.39.42-alpha ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index e041222e4ef3..a0218c6ac92e 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye ARG JDK_IMAGE=openjdk:${JDK_VERSION} FROM ${JDK_IMAGE} AS metrics-reporter -ARG VERSION=0.39.41-alpha +ARG VERSION=0.39.42-alpha ENV APPLICATION airbyte-metrics-reporter ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 8f30f7106b3d..3d59e086bc20 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -4,7 +4,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 -ARG VERSION=0.39.41-alpha +ARG VERSION=0.39.42-alpha ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 5d305e831ac6..15d820b1e91d 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "airbyte-webapp", - "version": "0.39.41-alpha", + "version": "0.39.42-alpha", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "airbyte-webapp", - "version": "0.39.41-alpha", + "version": "0.39.42-alpha", "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.1.1", "@fortawesome/free-brands-svg-icons": "^6.1.1", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index da0d0695f657..a65d4a5b0189 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.39.41-alpha", + "version": "0.39.42-alpha", "private": true, "engines": { "node": ">=16.0.0" diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 742fd1c0dbad..6a784341edd7 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -27,7 +27,7 @@ RUN wget -O /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages. RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list RUN apt-get update && apt-get install -y kubectl -ARG VERSION=0.39.41-alpha +ARG VERSION=0.39.42-alpha ENV APPLICATION airbyte-workers ENV VERSION ${VERSION} diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index c012fa854b1f..a59cc4c62ad5 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.39.41-alpha" +appVersion: "0.39.42-alpha" dependencies: - name: common diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index df7122d9b3b0..6abe50b80448 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.39.41-alpha" +appVersion: "0.39.42-alpha" dependencies: - name: common diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 49e5355f85e4..dd84d6e09b9e 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.39.41-alpha" +appVersion: "0.39.42-alpha" dependencies: - name: common diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 41b1d41bd566..d9a1d4aeb3fd 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.39.41-alpha" +appVersion: "0.39.42-alpha" dependencies: - name: common diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 57c5196a36b9..327365311e03 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.39.41-alpha" +appVersion: "0.39.42-alpha" dependencies: - name: common diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index be7d43241159..c4403b4ff5a8 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -22,7 +22,7 @@ version: 0.39.36 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.39.41-alpha" +appVersion: "0.39.42-alpha" dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index d4a8a4d054a5..2b4133f64149 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,6 @@ # airbyte -![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.39.41-alpha](https://img.shields.io/badge/AppVersion-0.39.41--alpha-informational?style=flat-square) +![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.39.42-alpha](https://img.shields.io/badge/AppVersion-0.39.41--alpha-informational?style=flat-square) Helm chart to deploy airbyte @@ -26,7 +26,7 @@ Helm chart to deploy airbyte | airbyte-bootloader.enabled | bool | `true` | | | airbyte-bootloader.image.pullPolicy | string | `"IfNotPresent"` | | | airbyte-bootloader.image.repository | string | `"airbyte/bootloader"` | | -| airbyte-bootloader.image.tag | string | `"0.39.41-alpha"` | | +| airbyte-bootloader.image.tag | string | `"0.39.42-alpha"` | | | airbyte-bootloader.nodeSelector | object | `{}` | | | airbyte-bootloader.podAnnotations | object | `{}` | | | airbyte-bootloader.resources.limits | object | `{}` | | @@ -113,7 +113,7 @@ Helm chart to deploy airbyte | server.extraVolumes | list | `[]` | | | server.image.pullPolicy | string | `"IfNotPresent"` | | | server.image.repository | string | `"airbyte/server"` | | -| server.image.tag | string | `"0.39.41-alpha"` | | +| server.image.tag | string | `"0.39.42-alpha"` | | | server.livenessProbe.enabled | bool | `true` | | | server.livenessProbe.failureThreshold | int | `3` | | | server.livenessProbe.initialDelaySeconds | int | `30` | | @@ -179,7 +179,7 @@ Helm chart to deploy airbyte | webapp.fullstory.enabled | bool | `false` | | | webapp.image.pullPolicy | string | `"IfNotPresent"` | | | webapp.image.repository | string | `"airbyte/webapp"` | | -| webapp.image.tag | string | `"0.39.41-alpha"` | | +| webapp.image.tag | string | `"0.39.42-alpha"` | | | webapp.ingress.annotations | object | `{}` | | | webapp.ingress.className | string | `""` | | | webapp.ingress.enabled | bool | `false` | | @@ -215,7 +215,7 @@ Helm chart to deploy airbyte | worker.extraVolumes | list | `[]` | | | worker.image.pullPolicy | string | `"IfNotPresent"` | | | worker.image.repository | string | `"airbyte/worker"` | | -| worker.image.tag | string | `"0.39.41-alpha"` | | +| worker.image.tag | string | `"0.39.42-alpha"` | | | worker.livenessProbe.enabled | bool | `true` | | | worker.livenessProbe.failureThreshold | int | `3` | | | worker.livenessProbe.initialDelaySeconds | int | `30` | | diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index f113cc52a9b7..135a7acc2b39 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -143,7 +143,7 @@ webapp: image: repository: airbyte/webapp pullPolicy: IfNotPresent - tag: 0.39.41-alpha + tag: 0.39.42-alpha ## @param webapp.podAnnotations [object] Add extra annotations to the webapp pod(s) ## @@ -419,7 +419,7 @@ server: image: repository: airbyte/server pullPolicy: IfNotPresent - tag: 0.39.41-alpha + tag: 0.39.42-alpha ## @param server.podAnnotations [object] Add extra annotations to the server pod ## @@ -547,7 +547,7 @@ worker: image: repository: airbyte/worker pullPolicy: IfNotPresent - tag: 0.39.41-alpha + tag: 0.39.42-alpha ## @param worker.podAnnotations [object] Add extra annotations to the worker pod(s) ## @@ -666,7 +666,7 @@ airbyte-bootloader: image: repository: airbyte/bootloader pullPolicy: IfNotPresent - tag: 0.39.41-alpha + tag: 0.39.42-alpha ## @param bootloader.podAnnotations [object] Add extra annotations to the bootloader pod ## diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index be7089365bf2..b68430d145d5 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -103,7 +103,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.39.41-alpha --\ + docker run --rm -v /tmp:/config airbyte/migration:0.39.42-alpha --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 0e536263ef5e..e1ee4a4fd38e 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.39.41-alpha +AIRBYTE_VERSION=0.39.42-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml index 0de5f53cffb7..4cc46d3640f3 100644 --- a/kube/overlays/stable-with-resource-limits/kustomization.yaml +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -8,15 +8,15 @@ bases: images: - name: airbyte/db - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/bootloader - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/server - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/webapp - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/worker - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 1fd6f49097d3..b23717ce6c2a 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.39.41-alpha +AIRBYTE_VERSION=0.39.42-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index 1b9175d9b696..df2a6cd20ecc 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -8,15 +8,15 @@ bases: images: - name: airbyte/db - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/bootloader - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/server - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/webapp - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: airbyte/worker - newTag: 0.39.41-alpha + newTag: 0.39.42-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index ad82453e13e8..95ef4e2eb3fd 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -14,5 +14,5 @@ USER octavia-cli WORKDIR /home/octavia-project ENTRYPOINT ["octavia"] -LABEL io.airbyte.version=0.39.41-alpha +LABEL io.airbyte.version=0.39.42-alpha LABEL io.airbyte.name=airbyte/octavia-cli diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 3076fa997c8e..50bf04b5a8d2 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -104,7 +104,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.41-alpha +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.42-alpha ``` ### Using `docker-compose` diff --git a/octavia-cli/install.sh b/octavia-cli/install.sh index a2443a252732..89d351e288bb 100755 --- a/octavia-cli/install.sh +++ b/octavia-cli/install.sh @@ -3,7 +3,7 @@ # This install scripts currently only works for ZSH and Bash profiles. # It creates an octavia alias in your profile bound to a docker run command and your current user. -VERSION=0.39.41-alpha +VERSION=0.39.42-alpha OCTAVIA_ENV_FILE=${HOME}/.octavia detect_profile() { diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index 0199983dfdaa..4246a7e34bba 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -15,7 +15,7 @@ setup( name="octavia-cli", - version="0.39.41", + version="0.39.42", description="A command line interface to manage Airbyte configurations", long_description=README, author="Airbyte", From c5c13f05b49c040da8494b7fdae96bf4c48c740f Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 5 Aug 2022 15:52:45 -0700 Subject: [PATCH 24/25] low-code-connectors: handle single records (#15346) * handle single records * comment * comment --- .../sources/declarative/extractors/record_selector.py | 3 +++ .../sources/declarative/extractors/test_record_selector.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index 8f27fb125b9c..dd738a69015d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -39,6 +39,9 @@ def select_records( next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: all_records = self.extractor.extract_records(response) + # Some APIs don't wrap single records in a list + if not isinstance(all_records, list): + all_records = [all_records] if self.record_filter: return self.record_filter.filter_records( all_records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index ed7aa35e6245..fa2bbfdcd7ce 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -36,6 +36,13 @@ {"data": [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}]}, [{"id": 3, "created_at": "06-08-21"}], ), + ( + "test_read_single_record", + "_.data", + None, + {"data": {"id": 1, "created_at": "06-06-21"}}, + [{"id": 1, "created_at": "06-06-21"}], + ), ], ) def test_record_filter(test_name, transform_template, filter_template, body, expected_records): From 5242ff8e95e58cad106f62d5f3b59a75c39ae940 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Fri, 5 Aug 2022 16:44:56 -0700 Subject: [PATCH 25/25] low-code connectors: reset pagination between stream slices (#15330) * reset pagination between stream slices * Update airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py Co-authored-by: Sherif A. Nada * Update airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py Co-authored-by: Sherif A. Nada * patch Co-authored-by: Sherif A. Nada --- .../requesters/paginators/limit_paginator.py | 3 +++ .../requesters/paginators/no_pagination.py | 4 ++++ .../declarative/requesters/paginators/paginator.py | 6 ++++++ .../strategies/cursor_pagination_strategy.py | 4 ++++ .../paginators/strategies/offset_increment.py | 3 +++ .../paginators/strategies/page_increment.py | 9 ++++++--- .../paginators/strategies/pagination_strategy.py | 6 ++++++ .../declarative/retrievers/simple_retriever.py | 1 + .../requesters/paginators/test_limit_paginator.py | 11 +++++++++++ .../requesters/paginators/test_offset_increment.py | 3 +++ .../requesters/paginators/test_page_increment.py | 7 +++++-- .../declarative/retrievers/test_simple_retriever.py | 13 ++++++++++--- 12 files changed, 62 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py index 675270f0da87..bf9adcbd515b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py @@ -147,6 +147,9 @@ def get_request_body_json( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.body_json) + def reset(self): + self.pagination_strategy.reset() + def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]: options = {} if self.page_token_option.inject_into == option_type: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py index ac54ba0bc70e..210b00c73123 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py @@ -59,3 +59,7 @@ def get_request_body_json( def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Mapping[str, Any]: return {} + + def reset(self): + # No state to reset + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py index e77ca744b3ed..68b18307e088 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py @@ -19,6 +19,12 @@ class Paginator(RequestOptionsProvider): If the next_page_token is the path to the next page of records, then it should be accessed through the `path` method """ + @abstractmethod + def reset(self): + """ + Reset the pagination's inner state + """ + @abstractmethod def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index 09d036580f8f..81577159735b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -46,3 +46,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin return None token = self.cursor_value.eval(config=self.config, last_records=last_records, response=decoded_response) return token if token else None + + def reset(self): + # No state to reset + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py index bfbd92df3e24..e6ab8a03fb58 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py @@ -31,3 +31,6 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin else: self._offset += len(last_records) return self._offset + + def reset(self): + self._offset = 0 diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index f39ca388ada1..46e112a0397f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -23,11 +23,14 @@ class PageIncrement(PaginationStrategy, JsonSchemaMixin): options: InitVar[Mapping[str, Any]] def __post_init__(self, options: Mapping[str, Any]): - self._offset = 0 + self._page = 0 def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: if len(last_records) < self.page_size: return None else: - self._offset += 1 - return self._offset + self._page += 1 + return self._page + + def reset(self): + self._page = 0 diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py index 7174fc16a377..a2d9407a833d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py @@ -24,3 +24,9 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin :return: next page token. Returns None if there are no more pages to fetch """ pass + + @abstractmethod + def reset(self): + """ + Reset the pagination's inner state + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 8eda1ec15401..4cfdbc8fd148 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -342,6 +342,7 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: # Warning: use self.state instead of the stream_state passed as argument! stream_slice = stream_slice or {} # None-check + self.paginator.reset() records_generator = HttpStream.read_records(self, sync_mode, cursor_field, stream_slice, self.state) for r in records_generator: self.stream_slicer.update_cursor(stream_slice, last_record=r) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py index cbdae4e48531..26d55a0276ee 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py @@ -3,6 +3,7 @@ # import json +from unittest.mock import MagicMock import pytest import requests @@ -159,3 +160,13 @@ def test_limit_cannot_be_set_in_path(): assert False except ValueError: pass + + +def test_reset(): + limit_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={}) + page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) + url_base = "https://airbyte.io" + config = {} + strategy = MagicMock() + LimitPaginator(2, limit_request_option, page_token_request_option, strategy, config, url_base, options={}).reset() + assert strategy.reset.called diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py index 7376ef155b43..c8f11a76ad60 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py @@ -30,3 +30,6 @@ def test_offset_increment_paginator_strategy(test_name, page_size, expected_next next_page_token = paginator_strategy.next_page_token(response, last_records) assert expected_next_page_token == next_page_token assert expected_offset == paginator_strategy._offset + + paginator_strategy.reset() + assert 0 == paginator_strategy._offset diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index fa3808a916b0..9d85cf8298b9 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -18,7 +18,7 @@ ) def test_page_increment_paginator_strategy(test_name, page_size, expected_next_page_token, expected_offset): paginator_strategy = PageIncrement(page_size, options={}) - assert paginator_strategy._offset == 0 + assert paginator_strategy._page == 0 response = requests.Response() @@ -29,4 +29,7 @@ def test_page_increment_paginator_strategy(test_name, page_size, expected_next_p next_page_token = paginator_strategy.next_page_token(response, last_records) assert expected_next_page_token == next_page_token - assert expected_offset == paginator_strategy._offset + assert expected_offset == paginator_strategy._page + + paginator_strategy.reset() + assert 0 == paginator_strategy._page diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index ad1ce696acb5..00d4e7fb7a87 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -2,7 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import airbyte_cdk.sources.declarative.requesters.error_handlers.response_status as response_status import pytest @@ -15,13 +15,15 @@ from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.streams.http.auth import NoAuth +from airbyte_cdk.sources.streams.http.http import HttpStream primary_key = "pk" records = [{"id": 1}, {"id": 2}] config = {} -def test_simple_retriever_full(): +@patch.object(HttpStream, "read_records", return_value=[]) +def test_simple_retriever_full(mock_http_stream): requester = MagicMock() request_params = {"param": "value"} requester.get_request_params.return_value = request_params @@ -53,6 +55,9 @@ def test_simple_retriever_full(): backoff_time = 60 should_retry = ResponseStatus.retry(backoff_time) requester.should_retry.return_value = should_retry + request_body_json = {"body": "json"} + requester.request_body_json.return_value = request_body_json + request_body_data = {"body": "data"} requester.get_request_body_data.return_value = request_body_data request_body_json = {"body": "json"} @@ -92,12 +97,14 @@ def test_simple_retriever_full(): assert not retriever.raise_on_http_errors assert retriever.should_retry(requests.Response()) assert retriever.backoff_time(requests.Response()) == backoff_time - assert retriever.request_body_data(None, None, None) == request_body_data assert retriever.request_body_json(None, None, None) == request_body_json assert retriever.request_kwargs(None, None, None) == request_kwargs assert retriever.cache_filename == cache_filename assert retriever.use_cache == use_cache + [r for r in retriever.read_records(SyncMode.full_refresh)] + paginator.reset.assert_called() + @pytest.mark.parametrize( "test_name, requester_response, expected_should_retry, expected_backoff_time",