From f21a4867e2ddb66d47d5e61f5d68eeba272fdbe1 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 22 Nov 2022 16:24:38 +0200 Subject: [PATCH 1/2] #CDK Lowcode: allow parametrizing initial page when using PageIncrement --- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../declarative/config_component_schema.json | 3 +++ .../paginators/strategies/page_increment.py | 6 ++++-- airbyte-cdk/python/setup.py | 2 +- .../requesters/paginators/test_page_increment.py | 16 +++++++++------- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index b37ad9f9520c..a2633a0cedb9 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.9.6 +Low-code: Add `start_from_page` option to a PageIncrement class + ## 0.9.5 Low-code: Add jinja macro `format_datetime` diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json index 8076c6f4ad7f..4d3341b8e2ea 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json @@ -1102,6 +1102,9 @@ "properties": { "page_size": { "type": "integer" + }, + "start_from_page": { + "type": "integer" } } } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index 530c8fbcfcc7..7f3ffaf207b5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -17,13 +17,15 @@ class PageIncrement(PaginationStrategy, JsonSchemaMixin): Attributes: page_size (int): the number of records to request + start_from_page (int): number of the initial page """ page_size: int options: InitVar[Mapping[str, Any]] + start_from_page: int = 0 def __post_init__(self, options: Mapping[str, Any]): - self._page = 0 + self._page = self.start_from_page def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: if len(last_records) < self.page_size: @@ -33,7 +35,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin return self._page def reset(self): - self._page = 0 + self._page = self.start_from_page def get_page_size(self) -> Optional[int]: return self.page_size diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 11222c1020ab..4b2a7c0f5008 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.9.5", + version="0.9.6", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index 9d85cf8298b9..699e0f794a50 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -10,15 +10,17 @@ @pytest.mark.parametrize( - "test_name, page_size, expected_next_page_token, expected_offset", + "test_name, page_size, start_from, expected_next_page_token, expected_offset", [ - ("test_same_page_size", 2, 1, 1), - ("test_larger_page_size", 3, None, 0), + ("test_same_page_size_start_from_0", 2, 1, 2, 2), + ("test_larger_page_size_start_from_0", 3, 1, None, 1), + ("test_same_page_size_start_from_1", 2, 0, 1, 1), + ("test_larger_page_size_start_from_0", 3, 0, None, 0) ], ) -def test_page_increment_paginator_strategy(test_name, page_size, expected_next_page_token, expected_offset): - paginator_strategy = PageIncrement(page_size, options={}) - assert paginator_strategy._page == 0 +def test_page_increment_paginator_strategy(test_name, page_size, start_from, expected_next_page_token, expected_offset): + paginator_strategy = PageIncrement(page_size, options={}, start_from_page=start_from) + assert paginator_strategy._page == start_from response = requests.Response() @@ -32,4 +34,4 @@ def test_page_increment_paginator_strategy(test_name, page_size, expected_next_p assert expected_offset == paginator_strategy._page paginator_strategy.reset() - assert 0 == paginator_strategy._page + assert start_from == paginator_strategy._page From d1593806f17ca6f2776962746eadb8bcd8cc66d5 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Wed, 23 Nov 2022 10:11:15 +0200 Subject: [PATCH 2/2] bump minor version --- airbyte-cdk/python/CHANGELOG.md | 2 +- airbyte-cdk/python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index a2633a0cedb9..2d011511854f 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.9.6 +## 0.10.0 Low-code: Add `start_from_page` option to a PageIncrement class ## 0.9.5 diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 4b2a7c0f5008..133323cb4827 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.9.6", + version="0.10.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown",