diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index 38780ac1e467..27ea60d469f0 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.1 +LABEL io.airbyte.version=2.0.2 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index 7d4c116e7a3d..f064522d0c5a 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -88,7 +88,9 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tests if the input configuration can be used to successfully connect to the integration """ try: - next(Advertisers(**self._prepare_stream_args(config)).read_records(SyncMode.full_refresh)) + advertisers = Advertisers(**self._prepare_stream_args(config)) + for slice_ in advertisers.stream_slices(): + next(advertisers.read_records(SyncMode.full_refresh, stream_slice=slice_)) except Exception as err: return False, err return True, None diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index f426710b7169..923564a6452c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -278,7 +278,7 @@ def __init__(self, start_date: str, end_date: str, **kwargs): def convert_array_param(arr: List[Union[str, int]]) -> str: return json.dumps(arr) - def get_advertiser_ids(self) -> Iterable[int]: + def get_advertiser_ids(self) -> List[int]: if self.is_sandbox: # for sandbox: just return advertiser_id provided in spec ids = [self._advertiser_id] @@ -306,9 +306,8 @@ def is_finished(self): def request_params( self, stream_state: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, - **kwargs, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = {"page_size": self.page_size} if self.fields: @@ -412,17 +411,23 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late class Advertisers(FullRefreshTiktokStream): """Docs: https://ads.tiktok.com/marketing_api/docs?id=1708503202263042""" - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(**kwargs) - params["advertiser_ids"] = self.convert_array_param(self.get_advertiser_ids()) - return params + def request_params( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + stream_slice = stream_slice or {} + return {key: self.convert_array_param(value) for key, value in stream_slice.items()} def path(self, *args, **kwargs) -> str: return "advertiser/info/" def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """this stream must work with the default slice logic""" - yield None + ids = self.get_advertiser_ids() + start, end, step = 0, len(ids), 100 + for i in range(start, end, step): + yield {"advertiser_ids": ids[i: min(end, i + step)]} class Campaigns(IncrementalTiktokStream): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py index 4453af2f9a77..af7333139544 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py @@ -78,9 +78,10 @@ def test_get_time_interval_past(pendulum_now_mock_past): assert len(list(intervals)) == 1 +@patch("source_tiktok_marketing.streams.AdvertiserIds.read_records", MagicMock(return_value=[{"advertiser_id": i} for i in range(354)])) def test_stream_slices_advertisers(): slices = Advertisers(**CONFIG).stream_slices() - assert list(slices) == [None] + assert len(list(slices)) == 4 # math.ceil(354 / 100) @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index ec992b7f890b..76ad025b0713 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -167,8 +167,9 @@ def logger_mock_fixture(): def test_source_check_connection_ok(config, logger_mock): - with patch.object(Advertisers, "read_records", return_value=iter([1])): - assert SourceTiktokMarketing().check_connection(logger_mock, config=config) == (True, None) + with patch.object(Advertisers, "stream_slices"): + with patch.object(Advertisers, "read_records", return_value=iter([1])): + assert SourceTiktokMarketing().check_connection(logger_mock, config=config) == (True, None) def test_source_check_connection_failed(config, logger_mock): diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index 53da4ce1b44c..5a53d88205f7 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -550,7 +550,8 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| -| 2.0.1 | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 2.0.2 | 2023-02-02 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Chunk Advertiser IDs | +| 2.0.1 | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 2.0.0 | 2022-12-20 | [20415](https://github.com/airbytehq/airbyte/pull/20415) | Update schema types for `AudienceReports` and `BasicReports` streams. | | 1.0.1 | 2022-12-16 | [20598](https://github.com/airbytehq/airbyte/pull/20598) | Remove Audience Reports with Hourly granularity due to deprecated dimension. | | 1.0.0 | 2022-12-05 | [19758](https://github.com/airbytehq/airbyte/pull/19758) | Convert `mobile_app_id` from integer to string in AudienceReport streams. |