🎉 source-facebook-marketing: allow configuration of page_size (#12171)

* source-facebook-marketing: allow configuration of page_size

* source-facebook-marketing: fix integration test for page_size conf

* fix: bin are added

* chore: update seed file

Co-authored-by: Harshith Mullapudi <harshithmullapudi@gmail.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -210,7 +210,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.45
+  dockerImageTag: 0.2.46
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -1781,7 +1781,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.45"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.46"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
@@ -2086,6 +2086,16 @@
                 format: "date-time"
             required:
             - "name"
+        page_size:
+          title: "Page Size of Requests"
+          description: "Page size used when sending requests to Facebook API to specify\
+            \ number of records per page when response has pagination. Most users\
+            \ do not need to set this field unless they specifically need to tune\
+            \ the connector to address specific issues or use cases."
+          default: 100
+          order: 7
+          exclusiveMinimum: 0
+          type: "integer"
       required:
       - "account_id"
       - "start_date"

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.2.45
+LABEL io.airbyte.version=0.2.46
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json

@@ -293,6 +293,14 @@
           },
           "required": ["name"]
         }
+      },
+      "page_size": {
+        "title": "Page Size of Requests",
+        "description": "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.",
+        "default": 100,
+        "order": 7,
+        "exclusiveMinimum": 0,
+        "type": "integer"
       }
     },
     "required": ["account_id", "start_date", "access_token"]

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py

@@ -67,20 +67,56 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
         )
         streams = [
             AdAccount(api=api),
-            AdSets(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted),
-            Ads(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted),
-            AdCreatives(api=api, fetch_thumbnail_images=config.fetch_thumbnail_images),
-            AdsInsights(**insights_args),
-            AdsInsightsAgeAndGender(**insights_args),
-            AdsInsightsCountry(**insights_args),
-            AdsInsightsRegion(**insights_args),
-            AdsInsightsDma(**insights_args),
-            AdsInsightsPlatformAndDevice(**insights_args),
-            AdsInsightsActionType(**insights_args),
-            Campaigns(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted),
-            Images(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted),
-            Videos(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted),
-            Activities(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted),
+            AdSets(
+                api=api,
+                start_date=config.start_date,
+                end_date=config.end_date,
+                include_deleted=config.include_deleted,
+                page_size=config.page_size,
+            ),
+            Ads(
+                api=api,
+                start_date=config.start_date,
+                end_date=config.end_date,
+                include_deleted=config.include_deleted,
+                page_size=config.page_size,
+            ),
+            AdCreatives(api=api, fetch_thumbnail_images=config.fetch_thumbnail_images, page_size=config.page_size),
+            AdsInsights(page_size=config.page_size, **insights_args),
+            AdsInsightsAgeAndGender(page_size=config.page_size, **insights_args),
+            AdsInsightsCountry(page_size=config.page_size, **insights_args),
+            AdsInsightsRegion(page_size=config.page_size, **insights_args),
+            AdsInsightsDma(page_size=config.page_size, **insights_args),
+            AdsInsightsPlatformAndDevice(page_size=config.page_size, **insights_args),
+            AdsInsightsActionType(page_size=config.page_size, **insights_args),
+            Campaigns(
+                api=api,
+                start_date=config.start_date,
+                end_date=config.end_date,
+                include_deleted=config.include_deleted,
+                page_size=config.page_size,
+            ),
+            Images(
+                api=api,
+                start_date=config.start_date,
+                end_date=config.end_date,
+                include_deleted=config.include_deleted,
+                page_size=config.page_size,
+            ),
+            Videos(
+                api=api,
+                start_date=config.start_date,
+                end_date=config.end_date,
+                include_deleted=config.include_deleted,
+                page_size=config.page_size,
+            ),
+            Activities(
+                api=api,
+                start_date=config.start_date,
+                end_date=config.end_date,
+                include_deleted=config.include_deleted,
+                page_size=config.page_size,
+            ),
         ]
 
         return self._update_insights_streams(insights=config.custom_insights, default_args=insights_args, streams=streams)

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py

@@ -147,3 +147,12 @@ class Config:
             "A list which contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)"
         ),
     )
+
+    page_size: Optional[PositiveInt] = Field(
+        title="Page Size of Requests",
+        order=7,
+        default=100,
+        description=(
+            "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases."
+        ),
+    )

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py

@@ -30,18 +30,17 @@ class FBMarketingStream(Stream, ABC):
     primary_key = "id"
     transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
 
-    # number of records per page when response has pagination
-    page_size = 100
     # use batch API to retrieve details for each record in a stream
     use_batch = True
     # this flag will override `include_deleted` option for streams that does not support it
     enable_deleted = True
     # entity prefix for `include_deleted` filter, it usually matches singular version of stream name
     entity_prefix = None
 
-    def __init__(self, api: "API", include_deleted: bool = False, **kwargs):
+    def __init__(self, api: "API", include_deleted: bool = False, page_size: int = 100, **kwargs):
         super().__init__(**kwargs)
         self._api = api
+        self.page_size = page_size if page_size is not None else 100
         self._include_deleted = include_deleted if self.enable_deleted else False
 
     @cached_property

docs/integrations/sources/facebook-marketing.md

@@ -108,6 +108,7 @@ For more information, see the [Facebook Insights API documentation.](https://dev
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.2.46  | 2022-04-22 | [12171](https://github.com/airbytehq/airbyte/pull/12171) | Allow configuration of page_size for requests |
 | 0.2.45  | 2022-05-03 | [12390](https://github.com/airbytehq/airbyte/pull/12390) | Better retry logic for split-up async jobs |
 | 0.2.44  | 2022-04-14 | [11751](https://github.com/airbytehq/airbyte/pull/11751) | Update API to a directly initialise an AdAccount with the given ID                                                                                                                                                                                                                                         |
 | 0.2.43  | 2022-04-13 | [11801](https://github.com/airbytehq/airbyte/pull/11801) | Fix `user_tos_accepted` schema to be an object