Skip to content

Commit

Permalink
🐛 [google-analytics-data-api] [tiktok-marketing] bump CDK version and…
Browse files Browse the repository at this point in the history
… fix state updater if no cursor_field (#36302)
  • Loading branch information
brianjlai authored Mar 25, 2024
1 parent a8fa18f commit 7528651
Show file tree
Hide file tree
Showing 14 changed files with 217 additions and 192 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a
dockerImageTag: 2.4.1
dockerImageTag: 2.4.2
dockerRepository: airbyte/source-google-analytics-data-api
documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-data-api
githubIssueLabel: source-google-analytics-data-api
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
build-backend = "poetry.core.masonry.api"

[tool.poetry]
version = "2.4.1"
version = "2.4.2"
name = "source-google-analytics-data-api"
description = "Source implementation for Google Analytics Data Api."
authors = [ "Airbyte <contact@airbyte.io>",]
Expand All @@ -19,7 +19,7 @@ include = "source_google_analytics_data_api"
python = "^3.9,<3.12"
cryptography = "==37.0.4"
requests = "==2.31.0"
airbyte-cdk = "==0.61.0"
airbyte-cdk = "^0"
PyJWT = "==2.4.0"
pandas = "==2.2.0"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,15 @@ def parse_response(
record["endDate"] = stream_slice["endDate"]
yield record

def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]):
def get_updated_state(
self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
) -> MutableMapping[str, Any]:
if not self.cursor_field:
# Some implementations of the GoogleAnalyticsDataApiBaseStream might not have a cursor because it's
# based on the `dimensions` config setting. This results in a full_refresh only stream that implements
# get_updated_state(), but does not define a cursor. For this scenario, there is no state value to extract
return {}

updated_state = (
utils.string_to_date(latest_record[self.cursor_field], self._record_date_format)
if self.cursor_field == "date"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ def test_migrate_config(capsys):
# migrate the test_config
migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE)

control_msg = json.loads(capsys.readouterr().out)
what = capsys.readouterr().out
control_msg = json.loads(what)
assert control_msg["type"] == Type.CONTROL.value
assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
{
"credentials": {
"auth_type": "Service",
"credentials_json": ""
},
"credentials": { "auth_type": "Service", "credentials_json": "" },
"custom_reports": "[{\"name\": \"custom_dimensions\", \"dimensions\": [\"date\", \"country\", \"device\"]}]",
"date_ranges_start_date": "2023-09-01",
"window_in_days": 30,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

import pytest
from freezegun import freeze_time
from source_google_analytics_data_api.source import GoogleAnalyticsDataApiBaseStream
from source_google_analytics_data_api.source import GoogleAnalyticsDataApiBaseStream, SourceGoogleAnalyticsDataApi

from .utils import read_incremental

Expand Down Expand Up @@ -391,3 +391,54 @@ def test_read_incremental(requests_mock):
{"property_id": 123, "yearWeek": "202202", "totalUsers": 125, "startDate": "2022-01-09", "endDate": "2022-01-09"},
{"property_id": 123, "yearWeek": "202202", "totalUsers": 140, "startDate": "2022-01-10", "endDate": "2022-01-10"},
]

@pytest.mark.parametrize(
"config_dimensions, expected_state",
[
pytest.param(["browser", "country", "language", "date"], {"date": "20240320"}, id="test_date_no_cursor_field_dimension"),
pytest.param(["browser", "country", "language"], {}, id="test_date_cursor_field_dimension"),
]
)
def test_get_updated_state(config_dimensions, expected_state):
config = {
"credentials": {
"auth_type": "Service",
"credentials_json": "{ \"client_email\": \"a@gmail.com\", \"client_id\": \"1234\", \"client_secret\": \"5678\", \"private_key\": \"5678\"}"
},
"date_ranges_start_date": "2023-04-01",
"window_in_days": 30,
"property_ids": ["123"],
"custom_reports_array": [
{
"name": "pivot_report",
"dateRanges": [{"startDate": "2020-09-01", "endDate": "2020-09-15"}],
"dimensions": config_dimensions,
"metrics": ["sessions"],
"pivots": [
{
"fieldNames": ["browser"],
"limit": 5
},
{
"fieldNames": ["country"],
"limit": 250
},
{
"fieldNames": ["language"],
"limit": 15
}
],
"cohortSpec": {
"enabled": "false"
}
}
]
}
source = SourceGoogleAnalyticsDataApi()
config = source._validate_and_transform(config, report_names=set())
config["authenticator"] = source.get_authenticator(config)
report_stream = source.instantiate_report_class(config["custom_reports_array"][0], False, config, page_size=100)

actual_state = report_stream.get_updated_state(current_stream_state={}, latest_record={"date": "20240320"})

assert actual_state == expected_state
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
dockerImageTag: 3.9.3
dockerImageTag: 3.9.4
dockerRepository: airbyte/source-tiktok-marketing
documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing
githubIssueLabel: source-tiktok-marketing
Expand Down
Loading

0 comments on commit 7528651

Please sign in to comment.