From 7536f7848bb2b28300a0e18c11c70893057a75fd Mon Sep 17 00:00:00 2001 From: FVidalCarneiro Date: Thu, 11 Jan 2024 15:43:17 +0100 Subject: [PATCH 1/8] fix: Changing next_page_token stopping criteria to when returned page from API is empty --- .../source-linkedin-ads/source_linkedin_ads/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py index 5151d52a961d..5f96f82bfcb1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py @@ -70,7 +70,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/pagination?context=linkedin/marketing/context """ parsed_response = response.json() - if len(parsed_response.get("elements")) < self.records_limit: + if len(parsed_response.get("elements")) == 0: return None return {"start": parsed_response.get("paging").get("start") + self.records_limit} From 61aa222540d96a953b304ebda4f106a33acf2608 Mon Sep 17 00:00:00 2001 From: FVidalCarneiro Date: Tue, 6 Feb 2024 14:13:18 +0100 Subject: [PATCH 2/8] feat: Complementing next page request loop stopping criteria --- .../source-linkedin-ads/source_linkedin_ads/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py index 5f96f82bfcb1..7f1c1c9f835d 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py @@ -70,7 +70,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/pagination?context=linkedin/marketing/context """ parsed_response = response.json() - if len(parsed_response.get("elements")) == 0: + if len(parsed_response.get("elements")) < self.records_limit and (parsed_response.get("paging")["start"] + self.records_limit > parsed_response.get("paging")["total"]): return None return {"start": parsed_response.get("paging").get("start") + self.records_limit} From b0b6a83ca56725aba0242f4ca51a11dbecc4968f Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 9 Feb 2024 01:02:58 +0100 Subject: [PATCH 3/8] Add chagelog and some refactoring --- .../source_linkedin_ads/analytics_streams.py | 8 +++++++- .../source-linkedin-ads/source_linkedin_ads/streams.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py index f58da0e25c8b..e25388da12a1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py @@ -211,7 +211,13 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, (See Restrictions: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-09&tabs=http#restrictions) """ parsed_response = response.json() - if len(parsed_response.get("elements")) < self.records_limit: + is_elements_less_than_limit = len(parsed_response.get("elements")) < self.records_limit + + # Note: The API might return fewer records than requested within the limits during pagination. + # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164 + is_end_of_records = parsed_response.get("paging")["start"] + self.records_limit > parsed_response.get("paging")["total"] + + if is_elements_less_than_limit and is_end_of_records: return None raise Exception( f"Limit {self.records_limit} elements exceeded. " diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py index 7f1c1c9f835d..e75479d83b8a 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py @@ -70,7 +70,13 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/pagination?context=linkedin/marketing/context """ parsed_response = response.json() - if len(parsed_response.get("elements")) < self.records_limit and (parsed_response.get("paging")["start"] + self.records_limit > parsed_response.get("paging")["total"]): + is_elements_less_than_limit = len(parsed_response.get("elements")) < self.records_limit + + # Note: The API might return fewer records than requested within the limits during pagination. + # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164 + is_end_of_records = parsed_response.get("paging")["start"] + self.records_limit > parsed_response.get("paging")["total"] + + if is_elements_less_than_limit and is_end_of_records: return None return {"start": parsed_response.get("paging").get("start") + self.records_limit} From d30bdf5991f04dfeaa778f6ac59ba1f47d137eaf Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 9 Feb 2024 10:57:28 +0100 Subject: [PATCH 4/8] Fix unittests and metadata --- .../source_linkedin_ads/analytics_streams.py | 7 ++++++- .../source-linkedin-ads/source_linkedin_ads/streams.py | 9 +++++++-- .../source-linkedin-ads/unit_tests/test_source.py | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py index e25388da12a1..f9ddc6aad742 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py @@ -215,7 +215,12 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, # Note: The API might return fewer records than requested within the limits during pagination. # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164 - is_end_of_records = parsed_response.get("paging")["start"] + self.records_limit > parsed_response.get("paging")["total"] + paging_params = parsed_response.get("paging", {}) + is_end_of_records = ( + paging_params["total"] - paging_params["start"] <= self.records_limit + if all(param in paging_params for param in ("total", "start")) + else True + ) if is_elements_less_than_limit and is_end_of_records: return None diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py index e75479d83b8a..e208df685d9f 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py @@ -74,11 +74,16 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, # Note: The API might return fewer records than requested within the limits during pagination. # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164 - is_end_of_records = parsed_response.get("paging")["start"] + self.records_limit > parsed_response.get("paging")["total"] + paging_params = parsed_response.get("paging", {}) + is_end_of_records = ( + paging_params["total"] - paging_params["start"] <= self.records_limit + if all(param in paging_params for param in ("total", "start")) + else True + ) if is_elements_less_than_limit and is_end_of_records: return None - return {"start": parsed_response.get("paging").get("start") + self.records_limit} + return {"start": paging_params.get("start") + self.records_limit} def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py index 56097007da27..09fe4da46afb 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py @@ -161,7 +161,7 @@ def test_accounts(self): "response_json, expected", ( ({"elements": []}, None), - ({"elements": [{"data": []}] * 500, "paging": {"start": 0}}, {"start": 500}), + ({"elements": [{"data": []}] * 500, "paging": {"start": 0, "total": 600}}, {"start": 500}), ), ) def test_next_page_token(self, requests_mock, response_json, expected): From a53f8f5a97691218bea574dc625b71e2c93c492b Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 9 Feb 2024 13:21:24 +0100 Subject: [PATCH 5/8] Re-run CAT From ce791c4c8187fed6537134205fd39fb315bcec48 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 9 Feb 2024 13:37:29 +0100 Subject: [PATCH 6/8] Re-run CAT From edcdbe5deff880b0b8d6db6e2a2ef895d21367db Mon Sep 17 00:00:00 2001 From: FVidalCarneiro Date: Tue, 27 Feb 2024 11:22:41 +0100 Subject: [PATCH 7/8] fix: Linkedin ads documentation --- .../connectors/source-linkedin-ads/pyproject.toml | 2 +- docs/integrations/sources/linkedin-ads.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml b/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml index bcbb55b92573..6fb4034f046b 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml +++ b/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.7.0" +version = "0.7.1" name = "source-linkedin-ads" description = "Source implementation for Linkedin Ads." authors = [ "Airbyte ",] diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index 858c297f7958..97d803b0e329 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -171,6 +171,7 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.7.1 | 2024-02-27 | [34152](https://github.com/airbytehq/airbyte/pull/34152) | Proceed pagination if return less than expected | | 0.7.0 | 2024-02-20 | [35465](https://github.com/airbytehq/airbyte/pull/35465) | Per-error reporting and continue sync on stream failures | | 0.6.8 | 2024-02-09 | [35086](https://github.com/airbytehq/airbyte/pull/35086) | Manage dependencies with Poetry. | | 0.6.7 | 2024-01-11 | [34152](https://github.com/airbytehq/airbyte/pull/34152) | prepare for airbyte-lib | From 7b7c168ffd94d644f6402e748036c5a47a927571 Mon Sep 17 00:00:00 2001 From: Natik Gadzhi Date: Thu, 18 Apr 2024 17:42:42 -0700 Subject: [PATCH 8/8] Bumping dockerImageTag to 1.0.1 for consistency. --- .../connectors/source-linkedin-ads/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml index 52e098a5d7d9..30950b7cf5e2 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 137ece28-5434-455c-8f34-69dc3782f451 - dockerImageTag: 1.0.0 + dockerImageTag: 1.0.1 dockerRepository: airbyte/source-linkedin-ads documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads githubIssueLabel: source-linkedin-ads