diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 88815b9885d4..5c9bd45a48c4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -65,7 +65,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 1.0.1 + dockerImageTag: 1.0.2 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 471cd825eb5d..1d027721d738 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -822,7 +822,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:1.0.1" +- dockerImage: "airbyte/source-amazon-ads:1.0.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index f44ce43716fc..d989493b2056 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,6 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 + +LABEL io.airbyte.version=1.0.2 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/README.md b/airbyte-integrations/connectors/source-amazon-ads/README.md index 610c77fb435d..acd73da4c15c 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/README.md +++ b/airbyte-integrations/connectors/source-amazon-ads/README.md @@ -81,7 +81,7 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` -pip install .[tests] +pip install .'[tests]' ``` ### Unit Tests To run unit tests locally, from the connector directory run: diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index 213a95432d9a..1a94dbe99854 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -42,20 +42,9 @@ acceptance_tests: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json - - config_path: secrets/config_report.json - configured_catalog_path: integration_tests/configured_catalog_report.json timeout_seconds: 3600 incremental: - tests: - - config_path: secrets/config_report.json - configured_catalog_path: integration_tests/configured_catalog_report.json - cursor_paths: - sponsored_products_report_stream: - - "1861552880916640" - - reportDate - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 2400 + bypass_reason: "can't populate stream because it requires real ad campaign" spec: tests: - spec_path: integration_tests/spec.json diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index c0a9cdcc27b7..036dc67d8258 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -3,47 +3,47 @@ # -from copy import copy +from http import HTTPStatus +from typing import List -from .report_streams import RecordType, ReportStream +from .report_streams import ReportInfo, ReportStream METRICS_MAP = { "campaigns": [ - "bidPlus", "campaignName", "campaignId", "campaignStatus", - "campaignBudget", - "campaignRuleBasedBudget", - "applicableBudgetRuleId", - "applicableBudgetRuleName", + "campaignBudgetAmount", + "campaignRuleBasedBudgetAmount", + "campaignApplicableBudgetRuleId", + "campaignApplicableBudgetRuleName", "impressions", "clicks", "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", ], "adGroups": [ "campaignName", @@ -53,30 +53,30 @@ "impressions", "clicks", "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", ], "keywords": [ "campaignName", @@ -84,161 +84,159 @@ "adGroupName", "adGroupId", "keywordId", - "keywordText", + "keyword", "matchType", "impressions", "clicks", "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", ], - "productAds": [ + "targets": [ "campaignName", "campaignId", "adGroupName", "adGroupId", - "adId", + "keywordId", + "keyword", + "targeting", + "keywordType", "impressions", "clicks", "cost", - "currency", - "asin", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", ], - "asins_keywords": [ + "productAds": [ "campaignName", "campaignId", "adGroupName", "adGroupId", - "keywordId", - "keywordText", "adId", - "asin", - "otherAsin", - "sku", - "currency", - "matchType", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedUnitsOrdered1dOtherSKU", - "attributedUnitsOrdered7dOtherSKU", - "attributedUnitsOrdered14dOtherSKU", - "attributedUnitsOrdered30dOtherSKU", - "attributedSales1dOtherSKU", - "attributedSales7dOtherSKU", - "attributedSales14dOtherSKU", - "attributedSales30dOtherSKU", + "impressions", + "clicks", + "cost", + "campaignBudgetCurrencyCode", + "advertisedAsin", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", ], - "asins_targets": [ + "asins_keywords": [ "campaignName", "campaignId", "adGroupName", "adGroupId", - "adId", - "asin", - "otherAsin", - "sku", - "currency", + "keywordId", + "keyword", + "advertisedAsin", + "purchasedAsin", + "advertisedSku", + "campaignBudgetCurrencyCode", "matchType", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedUnitsOrdered1dOtherSKU", - "attributedUnitsOrdered7dOtherSKU", - "attributedUnitsOrdered14dOtherSKU", - "attributedUnitsOrdered30dOtherSKU", - "attributedSales1dOtherSKU", - "attributedSales7dOtherSKU", - "attributedSales14dOtherSKU", - "attributedSales30dOtherSKU", - "targetId", - "targetingText", - "targetingType", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "unitsSoldOtherSku1d", + "unitsSoldOtherSku7d", + "unitsSoldOtherSku14d", + "unitsSoldOtherSku30d", + "salesOtherSku1d", + "salesOtherSku7d", + "salesOtherSku14d", + "salesOtherSku30d", ], - "targets": [ + "asins_targets": [ "campaignName", "campaignId", "adGroupName", "adGroupId", - "targetId", - "targetingExpression", - "targetingText", - "targetingType", - "impressions", - "clicks", - "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", + "advertisedAsin", + "purchasedAsin", + "advertisedSku", + "campaignBudgetCurrencyCode", + "matchType", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "unitsSoldOtherSku1d", + "unitsSoldOtherSku7d", + "unitsSoldOtherSku14d", + "unitsSoldOtherSku30d", + "salesOtherSku1d", + "salesOtherSku7d", + "salesOtherSku14d", + "salesOtherSku30d", + "keywordId", + "targeting", + "keywordType", ], } @@ -248,36 +246,74 @@ "adGroups": "adGroupId", "keywords": "keywordId", "productAds": "adId", - "asins_keywords": "asin", - "asins_targets": "asin", - "targets": "targetId", + "asins_keywords": "advertisedAsin", + "asins_targets": "advertisedAsin", + "targets": "keywordId", } class SponsoredProductsReportStream(ReportStream): """ https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Reports + https://advertising.amazon.com/API/docs/en-us/reporting/v3/migration-guide + https://advertising.amazon.com/API/docs/en-us/reporting/v3/report-types#sponsored-products """ - def report_init_endpoint(self, record_type: str) -> str: - return f"/v2/sp/{record_type}/report" - + API_VERSION = "reporting" # v3 + REPORT_DATE_FORMAT = "YYYY-MM-DD" + ad_product = "SPONSORED_PRODUCTS" + report_is_created = HTTPStatus.OK metrics_map = METRICS_MAP metrics_type_to_id_map = METRICS_TYPE_TO_ID_MAP + def report_init_endpoint(self, record_type: str) -> str: + return f"/{self.API_VERSION}/reports" + + def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: + """ + Download and parse report result + """ + return super()._download_report(None, url) + def _get_init_report_body(self, report_date: str, record_type: str, profile): metrics_list = self.metrics_map[record_type] + + reportTypeId = "spCampaigns" + group_by = ["campaign"] + filters = [] + + if record_type == "adGroups": + group_by.append("adGroup") + + elif record_type == "productAds": + reportTypeId = "spAdvertisedProduct" + group_by = ["advertiser"] + + elif "asin" in record_type: + reportTypeId = "spPurchasedProduct" + group_by = ["asin"] + + elif record_type == "keywords" or record_type == "targets": + group_by = ["targeting"] + reportTypeId = "spTargeting" + filters = [{"field": "keywordType", "values": ["TARGETING_EXPRESSION", "TARGETING_EXPRESSION_PREDEFINED"]}] + + if record_type == "keywords": + filters = [{"field": "keywordType", "values": ["BROAD", "PHRASE", "EXACT"]}] + body = { - "reportDate": report_date, + "name": f"{record_type} report {report_date}", + "startDate": report_date, + "endDate": report_date, + "configuration": { + "adProduct": self.ad_product, + "groupBy": group_by, + "columns": metrics_list, + "reportTypeId": reportTypeId, + "filters": filters, + "timeUnit": "SUMMARY", + "format": "GZIP_JSON", + }, } - if RecordType.ASINS in record_type: - body["campaignType"] = "sponsoredProducts" - if profile.accountInfo.type == "vendor": - metrics_list = copy(metrics_list) - metrics_list.remove("sku") - - # adId is automatically added to the report by amazon and requesting adId causes an amazon error - if "adId" in metrics_list: - metrics_list.remove("adId") - return {**body, "metrics": ",".join(metrics_list)} + return body diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 609c2f96ac51..324595719677 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -36,6 +36,7 @@ class RecordType(str, Enum): class Status(str, Enum): IN_PROGRESS = "IN_PROGRESS" SUCCESS = "SUCCESS" + COMPLETED = "COMPLETED" FAILURE = "FAILURE" @@ -47,6 +48,7 @@ class ReportInitResponse(BaseModel): class ReportStatus(BaseModel): status: str location: Optional[str] + url: Optional[str] @dataclass @@ -89,6 +91,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): Common base class for report streams """ + API_VERSION = "v2" primary_key = ["profileId", "recordType", "reportDate", "recordId"] # https://advertising.amazon.com/API/docs/en-us/reporting/v2/faq#what-is-the-available-report-history-for-the-version-2-reporting-api REPORTING_PERIOD = 60 @@ -96,6 +99,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "YYYYMMDD" cursor_field = "reportDate" + report_is_created = HTTPStatus.ACCEPTED ERRORS = [ (400, "KDP authors do not have access to Sponsored Brands functionality"), @@ -139,7 +143,7 @@ def read_records( collects metrics for all profiles and record types. Amazon Ads metric generation works in async way: First we need to initiate creating report for specific profile/record type/date and then constantly check for report - generation status - when it will have "SUCCESS" status then download the + generation status - when it will have "SUCCESS" or "COMPLETED" status then download the report and parse result. """ @@ -199,7 +203,7 @@ def _try_read_records(self, report_infos): if report_status == Status.FAILURE: message = f"Report for {report_info.profile_id} with {report_info.record_type} type generation failed" raise ReportGenerationFailure(message) - elif report_status == Status.SUCCESS: + elif report_status == Status.SUCCESS or report_status == Status.COMPLETED: try: report_info.metric_objects = self._download_report(report_info, download_url) except requests.HTTPError as error: @@ -211,7 +215,7 @@ def _try_read_records(self, report_infos): raise ReportGenerationInProgress(message) def _incomplete_report_infos(self, report_infos): - return [r for r in report_infos if r.status != Status.SUCCESS] + return [r for r in report_infos if r.status != Status.SUCCESS and r.status != Status.COMPLETED] def _generate_model(self): """ @@ -225,11 +229,15 @@ def _generate_model(self): return MetricsReport.generate_metric_model(metrics) def _get_auth_headers(self, profile_id: int): - return { - "Amazon-Advertising-API-ClientId": self._client_id, - "Amazon-Advertising-API-Scope": str(profile_id), - **self._authenticator.get_auth_header(), - } + return ( + { + "Amazon-Advertising-API-ClientId": self._client_id, + "Amazon-Advertising-API-Scope": str(profile_id), + **self._authenticator.get_auth_header(), + } + if profile_id + else {} + ) @abstractmethod def report_init_endpoint(self, record_type: str) -> str: @@ -257,7 +265,7 @@ def _check_status(self, report_info: ReportInfo) -> Tuple[Status, str]: """ Check report status and return download link if report generated successfuly """ - check_endpoint = f"/v2/reports/{report_info.report_id}" + check_endpoint = f"/{self.API_VERSION}/reports/{report_info.report_id}" resp = self._send_http_request(urljoin(self._url, check_endpoint), report_info.profile_id) try: @@ -265,7 +273,7 @@ def _check_status(self, report_info: ReportInfo) -> Tuple[Status, str]: except ValueError as error: raise ReportStatusFailure(error) - return resp.status, resp.location + return resp.status, resp.location or resp.url @backoff.on_exception( backoff.expo, @@ -383,7 +391,7 @@ def _init_reports(self, profile: Profile, report_date: str) -> List[ReportInfo]: profile.profileId, report_init_body, ) - if response.status_code != HTTPStatus.ACCEPTED: + if response.status_code != self.report_is_created: error_msg = f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}" if self._skip_known_errors(response): self.logger.warning(error_msg) @@ -413,7 +421,7 @@ def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: """ Download and parse report result """ - response = self._send_http_request(url, report_info.profile_id) + response = self._send_http_request(url, report_info.profile_id) if report_info else self._send_http_request(url, None) response.raise_for_status() raw_string = decompress(response.content).decode("utf") return json.loads(raw_string) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index c8afe17f3ee7..be13de943fbd 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -41,6 +41,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -51,6 +52,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -61,6 +63,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -71,6 +74,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -81,6 +85,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" } @@ -88,10 +93,12 @@ """ METRIC_RESPONSE = b64decode( """ -H4sIANnqymMC/92SsYrCQBBA+3zFsrWBmdnZ7K6lTbSRgyvFYjFBwl2iJIqI+O+3p2aPEyxSmmKLnceb4jHJKhHiEp -4QcuPrva+2zaKQU0HIYCyTnfyHS1+XAcsu/L/LtB+nTZinUZIPyxd5uzvubxtlxg5Q8R97jDOtCJB0Dw6+3ZaHOzQO -A1SM0eqq5hfkAPDxOUemnnyV59OuLWbVTdSIpNgZfsL3tS7TxioglAFeJy8aMGtgbWlIgt4ZRwENDpmtGnQFURpHA1 -JokGDYGURpHA2s0woyYBjSIErv1SBZJz+HyV3zFgUAAA== + H4sIAAAAAAAACuWSPWvDMBCGd/8KoTmGu9PJkrq1S5olBDqWDqIWwbRxgu + 02hJD/HjX+ooUMXutBg95Hzwle7jUR4hyPEPLd7w6+2JarXD4IQgZjmezi + N1z7XYhY1vH+GdI+TsuYp4MkO8vny2r/dbhNlBk7QMUj6+JMKwIk3YPGV9 + vQtNA4jFAxDlZdlD9gCQCbl2dkGud9h6op6pA/3n3zEU7HfZU/FbfhGpEU + O8N/cPu1y7SxCghlhJfFnZ6YNbC2NKWm3plPSxocMls1aZsGaT49kUKDBN + PWaZDm05N1WkEGDFN6sr30/3pK3q5AhIPlyAUAAA== """ ) METRICS_COUNT = 5 @@ -103,20 +110,36 @@ def setup_responses(init_response=None, init_response_products=None, init_respon if init_response_products: responses.add( responses.POST, - re.compile(r"https://advertising-api.amazon.com/v2/sp/[a-zA-Z]+/report"), + re.compile(r"https://advertising-api.amazon.com/reporting/reports"), body=init_response_products, - status=202, + status=200, + ) + responses.add( + responses.POST, + re.compile(r"https://advertising-api.amazon.com/reporting/reports"), + body=init_response_products, + status=200, ) if init_response_brands: responses.add( responses.POST, re.compile(r"https://advertising-api.amazon.com/v2/hsa/[a-zA-Z]+/report"), body=init_response_brands, status=202 ) if status_response: + responses.add( + responses.GET, + re.compile(r"https://advertising-api.amazon.com/reporting/reports/[^/]+$"), + body=status_response, + ) responses.add( responses.GET, re.compile(r"https://advertising-api.amazon.com/v2/reports/[^/]+$"), body=status_response, ) + responses.add( + responses.GET, + re.compile(r"https://advertising-api.amazon.com/reporting/reports"), + body=status_response, + ) if metric_response: responses.add( responses.GET, @@ -178,7 +201,7 @@ def test_products_report_stream(config): profiles = make_profiles(profile_type="vendor") stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"profile": profiles[0], "reportDate": "20210725", "retry_count": 3} + stream_slice = {"profile": profiles[0], "reportDate": "2021-07-25", "retry_count": 3} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) @@ -384,9 +407,9 @@ def test_get_start_date(config): profile_id = str(profiles[0].profileId) stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210810"}}) == Date(2021, 8, 10) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "2021-08-10"}}) == Date(2021, 8, 10) stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210510"}}) == Date(2021, 6, 1) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "2021-05-10"}}) == Date(2021, 6, 1) config.pop("start_date") stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) @@ -399,7 +422,7 @@ def test_stream_slices_different_timezones(config): profile2 = Profile(profileId=2, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) stream = SponsoredProductsReportStream(config, [profile1, profile2], authenticator=mock.MagicMock()) slices = list(stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state={})) - assert slices == [{"profile": profile1, "reportDate": "20210731"}, {"profile": profile2, "reportDate": "20210801"}] + assert slices == [{"profile": profile1, "reportDate": "2021-07-31"}, {"profile": profile2, "reportDate": "2021-08-01"}] def test_stream_slices_lazy_evaluation(config): @@ -417,19 +440,19 @@ def test_stream_slices_lazy_evaluation(config): frozen_datetime.tick(delta=timedelta(minutes=10)) assert slices == [ - {"profile": profile1, "reportDate": "20220527"}, - {"profile": profile2, "reportDate": "20220528"}, - {"profile": profile1, "reportDate": "20220528"}, - {"profile": profile2, "reportDate": "20220529"}, - {"profile": profile1, "reportDate": "20220529"}, - {"profile": profile2, "reportDate": "20220530"}, - {"profile": profile1, "reportDate": "20220530"}, - {"profile": profile2, "reportDate": "20220531"}, - {"profile": profile1, "reportDate": "20220531"}, - {"profile": profile2, "reportDate": "20220601"}, - {"profile": profile1, "reportDate": "20220601"}, - {"profile": profile2, "reportDate": "20220602"}, - {"profile": profile1, "reportDate": "20220602"}, + {"profile": profile1, "reportDate": "2022-05-27"}, + {"profile": profile2, "reportDate": "2022-05-28"}, + {"profile": profile1, "reportDate": "2022-05-28"}, + {"profile": profile2, "reportDate": "2022-05-29"}, + {"profile": profile1, "reportDate": "2022-05-29"}, + {"profile": profile2, "reportDate": "2022-05-30"}, + {"profile": profile1, "reportDate": "2022-05-30"}, + {"profile": profile2, "reportDate": "2022-05-31"}, + {"profile": profile1, "reportDate": "2022-05-31"}, + {"profile": profile2, "reportDate": "2022-06-01"}, + {"profile": profile1, "reportDate": "2022-06-01"}, + {"profile": profile2, "reportDate": "2022-06-02"}, + {"profile": profile1, "reportDate": "2022-06-02"}, ] @@ -438,10 +461,10 @@ def test_get_date_range_lazy_evaluation(): with freeze_time("2022-06-01T12:00:00+00:00") as frozen_datetime: date_range = list(get_date_range(start_date=Date(2022, 5, 29), timezone="UTC")) - assert date_range == ["20220529", "20220530", "20220531", "20220601"] + assert date_range == ["2022-05-29", "2022-05-30", "2022-05-31", "2022-06-01"] date_range = list(get_date_range(start_date=Date(2022, 6, 1), timezone="UTC")) - assert date_range == ["20220601"] + assert date_range == ["2022-06-01"] date_range = list(get_date_range(start_date=Date(2022, 6, 2), timezone="UTC")) assert date_range == [] @@ -450,7 +473,7 @@ def test_get_date_range_lazy_evaluation(): for date in get_date_range(start_date=Date(2022, 5, 29), timezone="UTC"): date_range.append(date) frozen_datetime.tick(delta=timedelta(hours=3)) - assert date_range == ["20220529", "20220530", "20220531", "20220601", "20220602"] + assert date_range == ["2022-05-29", "2022-05-30", "2022-05-31", "2022-06-01", "2022-06-02"] @responses.activate @@ -728,7 +751,7 @@ def test_products_report_stream_with_custom_record_types(config_gen, custom_reco profiles = make_profiles(profile_type="vendor") stream = SponsoredProductsReportStream(config_gen(report_record_types=custom_record_types), profiles, authenticator=mock.MagicMock()) - stream_slice = {"profile": profiles[0], "reportDate": "20210725", "retry_count": 3} + stream_slice = {"profile": profiles[0], "reportDate": "2021-07-25", "retry_count": 3} records = list(stream.read_records(SyncMode.incremental, stream_slice=stream_slice)) for record in records: print(record) diff --git a/connectors.md b/connectors.md index 0291c454b593..10099dcdcb80 100644 --- a/connectors.md +++ b/connectors.md @@ -11,7 +11,7 @@ | **Airtable** | Airtable icon | Source | airbyte/source-airtable:3.0.0 | generally_available | [link](https://docs.airbyte.com/integrations/sources/airtable) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-airtable) | `14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212` | | **AlloyDB for PostgreSQL** | AlloyDB for PostgreSQL icon | Source | airbyte/source-alloydb:2.0.13 | generally_available | [link](https://docs.airbyte.com/integrations/sources/alloydb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-alloydb) | `1fa90628-2b9e-11ed-a261-0242ac120002` | | **Alpha Vantage** | Alpha Vantage icon | Source | airbyte/source-alpha-vantage:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/alpha-vantage) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-alpha-vantage) | `db385323-9333-4fec-bec3-9e0ca9326c90` | -| **Amazon Ads** | Amazon Ads icon | Source | airbyte/source-amazon-ads:1.0.1 | generally_available | [link](https://docs.airbyte.com/integrations/sources/amazon-ads) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-ads) | `c6b0a29e-1da9-4512-9002-7bfd0cba2246` | +| **Amazon Ads** | Amazon Ads icon | Source | airbyte/source-amazon-ads:1.0.2 | generally_available | [link](https://docs.airbyte.com/integrations/sources/amazon-ads) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-ads) | `c6b0a29e-1da9-4512-9002-7bfd0cba2246` | | **Amazon SQS** | Amazon SQS icon | Source | airbyte/source-amazon-sqs:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/amazon-sqs) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-sqs) | `983fd355-6bf3-4709-91b5-37afa391eeb6` | | **Amazon Seller Partner** | Amazon Seller Partner icon | Source | airbyte/source-amazon-seller-partner:1.0.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/amazon-seller-partner) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-seller-partner) | `e55879a8-0ef8-4557-abcf-ab34c53ec460` | | **Amplitude** | Amplitude icon | Source | airbyte/source-amplitude:0.1.23 | generally_available | [link](https://docs.airbyte.com/integrations/sources/amplitude) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amplitude) | `fa9f58c6-2d03-4237-aaa4-07d75e0c1396` | diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 8018edaae343..048db1f8985f 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -94,6 +94,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 1.0.2 | 2023-02-03 | [22355](https://github.com/airbytehq/airbyte/pull/22355) | Migrate `products_report` stream to API v3 | | 1.0.1 | 2022-11-01 | [18677](https://github.com/airbytehq/airbyte/pull/18677) | Add optional config report_record_types | | 1.0.0 | 2023-01-30 | [21677](https://github.com/airbytehq/airbyte/pull/21677) | Fix bug with non-unique primary keys in report streams. Add asins_keywords and asins_targets | | 0.1.29 | 2023-01-27 | [22038](https://github.com/airbytehq/airbyte/pull/22038) | Set `AvailabilityStrategy` for streams explicitly to `None` |