Source Tiktok: allow to filter advertiser in reports (#23377)

* make possible to filter adverstiser id tiktok

* remove file

* make compatible with old config

* force true for old config

* make work with old config

* return sqlite file

* handle case assign variable is_sandbox

* add allowedHosts for Tiktok

* add allowedHosts

* fix tests

* auto-bump connector version

---------

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1917,7 +1917,7 @@
 - name: TikTok Marketing
   sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
   dockerRepository: airbyte/source-tiktok-marketing
-  dockerImageTag: 2.0.3
+  dockerImageTag: 2.0.4
   documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing
   icon: tiktok.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -15142,7 +15142,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tiktok-marketing:2.0.3"
+- dockerImage: "airbyte/source-tiktok-marketing:2.0.4"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/tiktok-marketing"
     changelogUrl: "https://docs.airbyte.com/integrations/sources/tiktok-marketing"
@@ -15180,6 +15180,11 @@
                 description: "Long-term Authorized Access Token."
                 airbyte_secret: true
                 type: "string"
+              advertiser_id:
+                title: "Advertiser ID"
+                description: "The Advertiser ID to filter reports and streams. Let\
+                  \ this empty to retrieve all."
+                type: "string"
             required:
             - "app_id"
             - "secret"

airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile

@@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=2.0.3
+LABEL io.airbyte.version=2.0.4
 LABEL io.airbyte.name=airbyte/source-tiktok-marketing

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json

@@ -39,6 +39,11 @@
                 "description": "Long-term Authorized Access Token.",
                 "airbyte_secret": true,
                 "type": "string"
+              },
+              "advertiser_id": {
+                "title": "Advertiser ID",
+                "description": "The Advertiser ID to filter reports and streams. Let this empty to retrieve all.",
+                "type": "string"
               }
             },
             "required": ["app_id", "secret", "access_token"]

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py

@@ -61,15 +61,20 @@ def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]:
         """Converts an input configure to stream arguments"""
 
         credentials = config.get("credentials")
+
         if credentials:
             # used for new config format
+            is_sandbox = credentials["auth_type"] == "sandbox_access_token"
             access_token = credentials["access_token"]
             secret = credentials.get("secret")
             app_id = int(credentials.get("app_id", 0))
             advertiser_id = int(credentials.get("advertiser_id", 0))
         else:
+            # old config only has advertiser id in environment object
+            # if there is a secret it is a prod config
             access_token = config["access_token"]
             secret = config.get("environment", {}).get("secret")
+            is_sandbox = secret is None
             app_id = int(config.get("environment", {}).get("app_id", 0))
             advertiser_id = int(config.get("environment", {}).get("advertiser_id", 0))
 
@@ -81,6 +86,7 @@ def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]:
             "app_id": app_id,
             "secret": secret,
             "access_token": access_token,
+            "is_sandbox": is_sandbox,
         }
 
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]:
@@ -98,7 +104,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         args = self._prepare_stream_args(config)
 
-        is_production = not (args["advertiser_id"])
+        is_production = not (args["is_sandbox"])
 
         report_granularity = config.get("report_granularity")
 

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json

@@ -39,6 +39,11 @@
                 "description": "Long-term Authorized Access Token.",
                 "airbyte_secret": true,
                 "type": "string"
+              },
+              "advertiser_id": {
+                "title": "Advertiser ID",
+                "description": "The Advertiser ID to filter reports and streams. Let this empty to retrieve all.",
+                "type": "string"
               }
             },
             "required": ["app_id", "secret", "access_token"]

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py

@@ -150,9 +150,7 @@ def __init__(self, **kwargs):
         super().__init__(authenticator=kwargs.get("authenticator"))
 
         self._advertiser_id = kwargs.get("advertiser_id")
-
-        # only sandbox has non-empty self._advertiser_id
-        self.is_sandbox = bool(self._advertiser_id)
+        self.is_sandbox = kwargs.get("is_sandbox")
 
     @property
     def availability_strategy(self) -> Optional["AvailabilityStrategy"]:
@@ -283,11 +281,13 @@ def convert_array_param(arr: List[Union[str, int]]) -> str:
         return json.dumps(arr)
 
     def get_advertiser_ids(self) -> List[int]:
-        if self.is_sandbox:
+        if self._advertiser_id:
             # for sandbox: just return advertiser_id provided in spec
+            # for production: it will filter only the advertiser id provied in spec
             ids = [self._advertiser_id]
         else:
-            # for prod: return list of all available ids from AdvertiserIds stream:
+            # for prod: return list of all available ids from AdvertiserIds stream if the field is empty
+            # in the connector configuration
             advertiser_ids = AdvertiserIds(**self.kwargs).read_records(sync_mode=SyncMode.full_refresh)
             ids = [advertiser["advertiser_id"] for advertiser in advertiser_ids]
 

connectors.md

@@ -233,7 +233,7 @@
 | **Tempo** | <img alt="Tempo icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/tempo.svg" height="30" height="30"/> | Source | airbyte/source-tempo:0.3.0 | beta | [link](https://docs.airbyte.com/integrations/sources/tempo) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tempo) | <small>`d1aa448b-7c54-498e-ad95-263cbebcd2db`</small> |
 | **The Guardian API** | <img alt="The Guardian API icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/theguardian.svg" height="30" height="30"/> | Source | airbyte/source-the-guardian-api:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/the-guardian-api) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-the-guardian-api) | <small>`d42bd69f-6bf0-4d0b-9209-16231af07a92`</small> |
 | **TiDB** | <img alt="TiDB icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/tidb.svg" height="30" height="30"/> | Source | airbyte/source-tidb:0.2.2 | alpha | [link](https://docs.airbyte.com/integrations/sources/tidb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tidb) | <small>`0dad1a35-ccf8-4d03-b73e-6788c00b13ae`</small> |
-| **TikTok Marketing** | <img alt="TikTok Marketing icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/tiktok.svg" height="30" height="30"/> | Source | airbyte/source-tiktok-marketing:2.0.3 | generally_available | [link](https://docs.airbyte.com/integrations/sources/tiktok-marketing) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tiktok-marketing) | <small>`4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35`</small> |
+| **TikTok Marketing** | <img alt="TikTok Marketing icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/tiktok.svg" height="30" height="30"/> | Source | airbyte/source-tiktok-marketing:2.0.4 | generally_available | [link](https://docs.airbyte.com/integrations/sources/tiktok-marketing) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tiktok-marketing) | <small>`4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35`</small> |
 | **Timely** | <img alt="Timely icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/timely.svg" height="30" height="30"/> | Source | airbyte/source-timely:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/timely) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-timely) | <small>`bc617b5f-1b9e-4a2d-bebe-782fd454a771`</small> |
 | **Toggl** | <img alt="Toggl icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/toggl.svg" height="30" height="30"/> | Source | airbyte/source-toggl:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/toggl) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-toggl) | <small>`7e7c844f-2300-4342-b7d3-6dd7992593cd`</small> |
 | **Trello** | <img alt="Trello icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/trello.svg" height="30" height="30"/> | Source | airbyte/source-trello:0.1.6 | alpha | [link](https://docs.airbyte.com/integrations/sources/trello) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-trello) | <small>`8da67652-004c-11ec-9a03-0242ac130003`</small> |

docs/integrations/sources/tiktok-marketing.md

@@ -550,6 +550,7 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark
 
 | Version | Date       | Pull Request                                             | Subject                                                                                       |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------|
+| 2.0.4   | 2023-02-23 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Add Advertiser ID to filter reports and streams |
 | 2.0.3   | 2023-02-15 | [23091](https://github.com/airbytehq/airbyte/pull/23091) | Add more clear log message for 504 error                                                      |
 | 2.0.2   | 2023-02-02 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Chunk Advertiser IDs                                                                          |
 | 2.0.1   | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None`                                   |