🚨🚨 Source Facebook Marketing: update API to v19.0

airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml

@@ -37,7 +37,7 @@ acceptance_tests:
   incremental:
     tests:
       - config_path: "secrets/config.json"
-        timeout_seconds: 4800
+        timeout_seconds: 6000
         future_state:
           future_state_path: "integration_tests/future_state.json"
   full_refresh:

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json

@@ -11,11 +11,11 @@
         "order": 0,
         "pattern_descriptor": "The Ad Account ID must be a number.",
         "examples": ["111111111111111"],
-        "type": "array",
         "minItems": 1,
+        "type": "array",
         "items": {
-          "pattern": "^[0-9]+$",
-          "type": "string"
+          "type": "string",
+          "pattern": "^[0-9]+$"
         },
         "uniqueItems": true
       },
@@ -175,6 +175,7 @@
                   "catalog_segment_value_omni_purchase_roas",
                   "catalog_segment_value_website_purchase_roas",
                   "clicks",
+                  "conversion_lead_rate",
                   "conversion_rate_ranking",
                   "conversion_values",
                   "conversions",
@@ -185,6 +186,7 @@
                   "cost_per_action_type",
                   "cost_per_ad_click",
                   "cost_per_conversion",
+                  "cost_per_conversion_lead",
                   "cost_per_dda_countby_convs",
                   "cost_per_estimated_ad_recallers",
                   "cost_per_inline_link_click",
@@ -229,6 +231,9 @@
                   "interactive_component_tap",
                   "labels",
                   "location",
+                  "marketing_messages_cost_per_delivered",
+                  "marketing_messages_cost_per_link_btn_click",
+                  "marketing_messages_spend",
                   "mobile_app_purchase_roas",
                   "objective",
                   "optimization_goal",
@@ -238,9 +243,6 @@
                   "purchase_roas",
                   "qualifying_question_qualify_answer_rate",
                   "quality_ranking",
-                  "quality_score_ectr",
-                  "quality_score_ecvr",
-                  "quality_score_organic",
                   "reach",
                   "social_spend",
                   "spend",
@@ -309,7 +311,16 @@
                   "image_asset",
                   "impression_device",
                   "is_conversion_id_modeled",
+                  "landing_destination",
                   "link_url_asset",
+                  "marketing_messages_btn_name",
+                  "mdsa_landing_destination",
+                  "media_asset_url",
+                  "media_creator",
+                  "media_destination_url",
+                  "media_format",
+                  "media_origin_url",
+                  "media_text_content",
                   "mmm",
                   "place_page_id",
                   "platform_position",
@@ -320,6 +331,8 @@
                   "region",
                   "skan_campaign_id",
                   "skan_conversion_id",
+                  "skan_version",
+                  "standard_event_content_type",
                   "title_asset",
                   "video_asset"
                 ]
@@ -343,7 +356,8 @@
                   "action_target_id",
                   "action_type",
                   "action_video_sound",
-                  "action_video_type"
+                  "action_video_type",
+                  "standard_event_content_type"
                 ]
               }
             },

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/test_streams.py

@@ -31,11 +31,7 @@ def configured_catalog_fixture(config) -> ConfiguredAirbyteCatalog:
         streams = []
         # Prefer incremental if available
         for stream in catalog.streams:
-            sync_mode = (
-                SyncMode.incremental
-                if SyncMode.incremental in stream.supported_sync_modes
-                else SyncMode.full_refresh
-            )
+            sync_mode = SyncMode.incremental if SyncMode.incremental in stream.supported_sync_modes else SyncMode.full_refresh
             streams.append(
                 ConfiguredAirbyteStream(
                     stream=stream,
@@ -56,9 +52,7 @@ class TestFacebookMarketingSource:
             ("ad_sets", "23846541706990398"),
         ],
     )
-    def test_streams_with_include_deleted(
-        self, stream_name, deleted_id, config_with_include_deleted, configured_catalog
-    ):
+    def test_streams_with_include_deleted(self, stream_name, deleted_id, config_with_include_deleted, configured_catalog):
         catalog = self._slice_catalog(configured_catalog, {stream_name})
         records, states = self._read_records(config_with_include_deleted, catalog)
         deleted_records = list(filter(self._deleted_record, records))
@@ -67,16 +61,10 @@ def test_streams_with_include_deleted(
 
         assert states, "incremental read should produce states"
         for name, state in states[-1].state.data.items():
-            assert (
-                "filter_statuses" in state[account_id]
-            ), f"State for {name} should include `filter_statuses` flag"
+            assert "filter_statuses" in state[account_id], f"State for {name} should include `filter_statuses` flag"
 
-        assert (
-            deleted_records
-        ), f"{stream_name} stream should have deleted records returned"
-        assert (
-            is_specific_deleted_pulled
-        ), f"{stream_name} stream should have a deleted record with id={deleted_id}"
+        assert deleted_records, f"{stream_name} stream should have deleted records returned"
+        assert is_specific_deleted_pulled, f"{stream_name} stream should have a deleted record with id={deleted_id}"
 
     @pytest.mark.parametrize(
         "stream_name, deleted_num, filter_statuses",
@@ -146,14 +134,10 @@ def test_streams_with_include_deleted_and_state(
                 value["filter_statuses"] = filter_statuses
 
         catalog = self._slice_catalog(configured_catalog, {stream_name})
-        records, states = self._read_records(
-            config_with_include_deleted, catalog, state=state
-        )
+        records, states = self._read_records(config_with_include_deleted, catalog, state=state)
         deleted_records = list(filter(self._deleted_record, records))
 
-        assert (
-            len(deleted_records) == deleted_num
-        ), f"{stream_name} should have {deleted_num} deleted records returned"
+        assert len(deleted_records) == deleted_num, f"{stream_name} should have {deleted_num} deleted records returned"
 
     @staticmethod
     def _deleted_record(record: AirbyteMessage) -> bool:
@@ -164,24 +148,18 @@ def _object_id(record: AirbyteMessage) -> str:
         return str(record.record.data["id"])
 
     @staticmethod
-    def _slice_catalog(
-        catalog: ConfiguredAirbyteCatalog, streams: Set[str]
-    ) -> ConfiguredAirbyteCatalog:
+    def _slice_catalog(catalog: ConfiguredAirbyteCatalog, streams: Set[str]) -> ConfiguredAirbyteCatalog:
         sliced_catalog = ConfiguredAirbyteCatalog(streams=[])
         for stream in catalog.streams:
             if stream.stream.name in streams:
                 sliced_catalog.streams.append(stream)
         return sliced_catalog
 
     @staticmethod
-    def _read_records(
-        conf, catalog, state=None
-    ) -> Tuple[List[AirbyteMessage], List[AirbyteMessage]]:
+    def _read_records(conf, catalog, state=None) -> Tuple[List[AirbyteMessage], List[AirbyteMessage]]:
         records = []
         states = []
-        for message in SourceFacebookMarketing().read(
-            logging.getLogger("airbyte"), conf, catalog, state=state
-        ):
+        for message in SourceFacebookMarketing().read(logging.getLogger("airbyte"), conf, catalog, state=state):
             if message.type == Type.RECORD:
                 records.append(message)
             elif message.type == Type.STATE:

airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml

@@ -6,11 +6,11 @@ data:
     hosts:
       - graph.facebook.com
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
+    baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 1.4.2
+  dockerImageTag: 2.0.0
   dockerRepository: airbyte/source-facebook-marketing
   documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing
   githubIssueLabel: source-facebook-marketing
@@ -27,6 +27,22 @@ data:
     oss:
       enabled: true
   releaseStage: generally_available
+  releases:
+    breakingChanges:
+      2.0.0:
+        message: "All Ads-Insights-* streams now have updated schemas. Users will need to retest source confguration, refresh the source schema and reset affected streams after upgrading. For more information [visit](https://docs.airbyte.com/integrations/sources/facebook-marketing-migrations)"
+        upgradeDeadline: "2024-03-17"
+        scopedImpact:
+          - scopeType: stream
+            impactedScopes:
+              - "ads_insights"
+              - "ads_insights_age_and_gender"
+              - "ads_insights_action_type"
+              - "ads_insights_country"
+              - "ads_insights_platform_and_device"
+              - "ads_insights_region"
+              - "ads_insights_dma"
+              - "ads_insights_action_product_id"
   suggestedStreams:
     streams:
       - ads_insights

airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.4.2"
+version = "2.0.0"
 name = "source-facebook-marketing"
 description = "Source implementation for Facebook Marketing."
 authors = [ "Airbyte <contact@airbyte.io>",]
@@ -18,9 +18,8 @@ include = "source_facebook_marketing"
 [tool.poetry.dependencies]
 python = "^3.9,<3.12"
 airbyte-cdk = "==0.62.1"
-facebook-business = "==17.0.0"
+facebook-business = "19.0.0"
 cached-property = "==1.5.2"
-pendulum = "==2.1.2"
 
 [tool.poetry.scripts]
 source-facebook-marketing = "source_facebook_marketing.run:run"

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json

@@ -33,9 +33,6 @@
     "adset_name": {
       "type": ["null", "string"]
     },
-    "age_targeting": {
-      "type": ["null", "string"]
-    },
     "attribution_setting": {
       "type": ["null", "string"]
     },
@@ -81,6 +78,9 @@
     "clicks": {
       "type": ["null", "integer"]
     },
+    "conversion_lead_rate": {
+      "type": ["null", "number"]
+    },
     "conversion_rate_ranking": {
       "type": ["null", "string"]
     },
@@ -111,6 +111,9 @@
     "cost_per_conversion": {
       "$ref": "ads_action_stats.json"
     },
+    "cost_per_conversion_lead": {
+      "type": ["null", "number"]
+    },
     "cost_per_estimated_ad_recallers": {
       "type": ["null", "number"]
     },
@@ -165,24 +168,9 @@
     "engagement_rate_ranking": {
       "type": ["null", "string"]
     },
-    "estimated_ad_recall_rate": {
-      "type": ["null", "number"]
-    },
-    "estimated_ad_recall_rate_lower_bound": {
-      "type": ["null", "number"]
-    },
-    "estimated_ad_recall_rate_upper_bound": {
-      "type": ["null", "number"]
-    },
     "estimated_ad_recallers": {
       "type": ["null", "number"]
     },
-    "estimated_ad_recallers_lower_bound": {
-      "type": ["null", "number"]
-    },
-    "estimated_ad_recallers_upper_bound": {
-      "type": ["null", "number"]
-    },
     "frequency": {
       "type": ["null", "number"]
     },
@@ -192,9 +180,6 @@
     "full_view_reach": {
       "type": ["null", "number"]
     },
-    "gender_targeting": {
-      "type": ["null", "string"]
-    },
     "impressions": {
       "type": ["null", "integer"]
     },
@@ -216,12 +201,6 @@
     "instant_experience_outbound_clicks": {
       "$ref": "ads_action_stats.json"
     },
-    "labels": {
-      "type": ["null", "string"]
-    },
-    "location": {
-      "type": ["null", "string"]
-    },
     "mobile_app_purchase_roas": {
       "$ref": "ads_action_stats.json"
     },

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py

@@ -70,8 +70,7 @@ def api_fixture(some_config, requests_mock, fb_account_response):
     )
     requests_mock.register_uri(
         "GET",
-        FacebookSession.GRAPH
-        + f"/{FB_API_VERSION}/act_{some_config['account_ids'][0]}/",
+        FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{some_config['account_ids'][0]}/",
         [fb_account_response],
     )
     return api

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py

@@ -70,7 +70,7 @@ def with_body(self, body: Union[str, bytes, Mapping[str, Any]]) -> RequestBuilde
 
     def build(self) -> HttpRequest:
         return HttpRequest(
-            url=f"https://graph.facebook.com/v17.0/{self._account_sub_path()}{self._resource}",
+            url=f"https://graph.facebook.com/v19.0/{self._account_sub_path()}{self._resource}",
             query_params=self._query_params,
             body=self._body,
         )