[run CI] Source Linkedin Ads: fix changing next_page_token stopping criteria

airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml

@@ -21,8 +21,7 @@ data:
   name: LinkedIn Ads
   remoteRegistries:
     pypi:
-      enabled: false
-      # TODO: Set enabled=true after `airbyte-lib-validate-source` is passing.
+      enabled: true
       packageName: airbyte-source-linkedin-ads
   registries:
     cloud:

airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py

@@ -211,7 +211,18 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
         (See Restrictions: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-09&tabs=http#restrictions)
         """
         parsed_response = response.json()
-        if len(parsed_response.get("elements")) < self.records_limit:
+        is_elements_less_than_limit = len(parsed_response.get("elements")) < self.records_limit
+
+        # Note: The API might return fewer records than requested within the limits during pagination.
+        # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164
+        paging_params = parsed_response.get("paging", {})
+        is_end_of_records = (
+            paging_params["total"] - paging_params["start"] <= self.records_limit
+            if all(param in paging_params for param in ("total", "start"))
+            else True
+        )
+
+        if is_elements_less_than_limit and is_end_of_records:
             return None
         raise Exception(
             f"Limit {self.records_limit} elements exceeded. "

airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py

@@ -70,9 +70,20 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
         https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/pagination?context=linkedin/marketing/context
         """
         parsed_response = response.json()
-        if len(parsed_response.get("elements")) < self.records_limit:
+        is_elements_less_than_limit = len(parsed_response.get("elements")) < self.records_limit
+
+        # Note: The API might return fewer records than requested within the limits during pagination.
+        # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164
+        paging_params = parsed_response.get("paging", {})
+        is_end_of_records = (
+            paging_params["total"] - paging_params["start"] <= self.records_limit
+            if all(param in paging_params for param in ("total", "start"))
+            else True
+        )
+
+        if is_elements_less_than_limit and is_end_of_records:
             return None
-        return {"start": parsed_response.get("paging").get("start") + self.records_limit}
+        return {"start": paging_params.get("start") + self.records_limit}
 
     def request_headers(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None

airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py

@@ -161,7 +161,7 @@ def test_accounts(self):
         "response_json, expected",
         (
             ({"elements": []}, None),
-            ({"elements": [{"data": []}] * 500, "paging": {"start": 0}}, {"start": 500}),
+            ({"elements": [{"data": []}] * 500, "paging": {"start": 0, "total": 600}}, {"start": 500}),
         ),
     )
     def test_next_page_token(self, requests_mock, response_json, expected):

docs/integrations/sources/linkedin-ads.md

@@ -170,7 +170,8 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
-| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- |
+|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 0.6.9   | 2024-02-09 | [34152](https://github.com/airbytehq/airbyte/pull/34152) | Proceed pagination if return less than expected                                                                 |
 | 0.6.8   | 2024-02-09 | [35086](https://github.com/airbytehq/airbyte/pull/35086) | Manage dependencies with Poetry.                                                                                |
 | 0.6.7   | 2024-01-11 | [34152](https://github.com/airbytehq/airbyte/pull/34152) | prepare for airbyte-lib                                                                                         |
 | 0.6.6   | 2024-01-15 | [34222](https://github.com/airbytehq/airbyte/pull/34222) | Use stream slices for Analytics streams                                                                         |