From bea82f2267a9642091cfc04ae2f91ca9136cbd93 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Fri, 19 Aug 2022 18:38:48 +0300 Subject: [PATCH] Source Facebook Marketing: retry FacebookBadObjectError (#15788) * #404 oncall: source facebook marketing - retry 'facebook bad object error' * source facebook marketing: upd changelog * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 2 +- .../source-facebook-marketing/Dockerfile | 2 +- .../source_facebook_marketing/streams/async_job.py | 8 +++++++- .../unit_tests/test_async_job.py | 14 +++++++++++++- docs/integrations/sources/facebook-marketing.md | 3 ++- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 6553a1437528..74809392f053 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -256,7 +256,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.59 + dockerImageTag: 0.2.60 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index da3399c52a0e..5494616903ac 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1857,7 +1857,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.59" +- dockerImage: "airbyte/source-facebook-marketing:0.2.60" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 1a77bfb5be8f..ee7b959c1e81 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.59 +LABEL io.airbyte.version=0.2.60 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py index 2f307c4fc9f7..dca8551dce30 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py @@ -8,6 +8,7 @@ from enum import Enum from typing import Any, Iterator, List, Mapping, Optional, Type, Union +import backoff import pendulum from facebook_business.adobjects.ad import Ad from facebook_business.adobjects.adaccount import AdAccount @@ -15,11 +16,15 @@ from facebook_business.adobjects.adset import AdSet from facebook_business.adobjects.campaign import Campaign from facebook_business.adobjects.objectparser import ObjectParser -from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookResponse +from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookBadObjectError, FacebookResponse +from source_facebook_marketing.streams.common import retry_pattern logger = logging.getLogger("airbyte") +backoff_policy = retry_pattern(backoff.expo, FacebookBadObjectError, max_tries=5, factor=5) + + def update_in_batch(api: FacebookAdsApi, jobs: List["AsyncJob"]): """Update status of each job in the list in a batch, making it most efficient way to update status. @@ -338,6 +343,7 @@ def _check_status(self) -> bool: return False + @backoff_policy def get_result(self) -> Any: """Retrieve result of the finished job.""" if not self._job or self.failed: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py index 31225d5a6a21..84c74c562c91 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py @@ -14,7 +14,7 @@ from facebook_business.adobjects.adset import AdSet from facebook_business.adobjects.adsinsights import AdsInsights from facebook_business.adobjects.campaign import Campaign -from facebook_business.api import FacebookAdsApiBatch +from facebook_business.api import FacebookAdsApiBatch, FacebookBadObjectError from source_facebook_marketing.api import MyFacebookAdsApi from source_facebook_marketing.streams.async_job import InsightAsyncJob, ParentAsyncJob, Status, update_in_batch @@ -299,6 +299,18 @@ def test_get_result(self, job, adreport, api): assert result[0].export_all_data() == {"some_data": 123} assert result[1].export_all_data() == {"some_data": 77} + def test_get_result_retried(self, mocker, job, api): + job.start() + api.call().json.return_value = {"data": [{"some_data": 123}, {"some_data": 77}]} + ads_insights = AdsInsights(api=api) + ads_insights._set_data({"items": [{"some_data": 123}, {"some_data": 77}]}) + with mocker.patch( + "facebook_business.adobjects.objectparser.ObjectParser.parse_multiple", + side_effect=[FacebookBadObjectError("Bad data to set object data"), ads_insights], + ): + # in case this is not retried, an error will be raised + job.get_result() + def test_get_result_when_job_is_not_started(self, job): with pytest.raises(RuntimeError, match=r"Incorrect usage of get_result - the job is not started or failed"): job.get_result() diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index d92ded0240b4..84021ee296fd 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -120,7 +120,8 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.2.59 | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method | +| 0.2.60 | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError | +| 0.2.59 | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method | | 0.2.58 | 2022-07-25 | [15012](https://github.com/airbytehq/airbyte/pull/15012) | Add `DATA_RETENTION_PERIOD`validation and fix `failed_delivery_checks` field schema type issue | | 0.2.57 | 2022-07-25 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Update Facebook SDK to version 14.0.0 | | 0.2.56 | 2022-07-19 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Add future `start_date` and `end_date` validation |