From 96a57eb172d0da0228d4844097eedee1d0566908 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 27 Oct 2022 21:50:24 +0300 Subject: [PATCH 1/3] validate custom_reports Signed-off-by: Sergey Chvalyuk --- .../source-google-search-console/Dockerfile | 2 +- .../source_google_search_console/source.py | 4 ++++ .../source_google_search_console/streams.py | 19 ++++++++++--------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile index 1276245292c8..c48b29620b6f 100755 --- a/airbyte-integrations/connectors/source-google-search-console/Dockerfile +++ b/airbyte-integrations/connectors/source-google-search-console/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-google-search-console diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py index 06729e9db31d..d05822969481 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py @@ -63,6 +63,10 @@ def _validate_and_transform(self, config: Mapping[str, Any]): except ValueError: raise Exception("custom_reports is not valid JSON") jsonschema.validate(config["custom_reports"], custom_reports_schema) + for report in config["custom_reports"]: + for dimension in report["dimensions"]: + if dimension not in SearchAnalyticsByCustomDimensions.dimension_to_property_schema_map: + raise Exception(f"dimension: '{dimension}' not found") pendulum.parse(config["start_date"]) end_date = config.get("end_date") diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index 3ba2adb1fa07..c52d3b0599b2 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -298,6 +298,14 @@ class SearchAnalyticsAllFields(SearchAnalytics): class SearchAnalyticsByCustomDimensions(SearchAnalytics): + dimension_to_property_schema_map = { + "country": [{"country": {"type": ["null", "string"]}}], + "date": [], + "device": [{"device": {"type": ["null", "string"]}}], + "page": [{"page": {"type": ["null", "string"]}}], + "query": [{"query": {"type": ["null", "string"]}}], + } + def __init__(self, dimensions: List[str], *args, **kwargs): super(SearchAnalyticsByCustomDimensions, self).__init__(*args, **kwargs) self.dimensions = dimensions @@ -305,7 +313,7 @@ def __init__(self, dimensions: List[str], *args, **kwargs): def get_json_schema(self) -> Mapping[str, Any]: try: return super(SearchAnalyticsByCustomDimensions, self).get_json_schema() - except IOError: + except FileNotFoundError: schema: Mapping[str, Any] = { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], @@ -327,16 +335,9 @@ def get_json_schema(self) -> Mapping[str, Any]: return schema def dimension_to_property_schema(self) -> dict: - dimension_to_property_schema_map = { - "country": [{"country": {"type": ["null", "string"]}}], - "date": [], - "device": [{"device": {"type": ["null", "string"]}}], - "page": [{"page": {"type": ["null", "string"]}}], - "query": [{"query": {"type": ["null", "string"]}}], - } properties = {} for dimension in sorted(self.dimensions): - fields = dimension_to_property_schema_map[dimension] + fields = self.dimension_to_property_schema_map[dimension] for field in fields: properties = {**properties, **field} return properties From 9eeed677869d9762749c676b525a176afd67f084 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 27 Oct 2022 21:54:46 +0300 Subject: [PATCH 2/3] google-search-console.md updated Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/google-search-console.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/google-search-console.md b/docs/integrations/sources/google-search-console.md index c8d0074504a9..41e6ca0e87f6 100644 --- a/docs/integrations/sources/google-search-console.md +++ b/docs/integrations/sources/google-search-console.md @@ -128,6 +128,7 @@ This connector attempts to back off gracefully when it hits Reports API's rate l | Version | Date | Pull Request | Subject | | :------- | :--------- | :------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------- | +| `0.1.18` | 2022-10-27 | [18568](https://github.com/airbytehq/airbyte/pull/18568) | Improved config validation: custom_reports.dimension | | `0.1.17` | 2022-10-08 | [17751](https://github.com/airbytehq/airbyte/pull/17751) | Improved config validation: start_date, end_date, site_urls | | `0.1.16` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | | `0.1.15` | 2022-09-16 | [16819](https://github.com/airbytehq/airbyte/pull/16819) | Check available site urls to avoid 403 error on sync | From 94fec26bd79b3ddc725bf8d28483ab63a8bda195 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 28 Oct 2022 05:42:03 +0000 Subject: [PATCH 3/3] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4e51df39a3a9..693c626ff4de 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -460,7 +460,7 @@ - name: Google Search Console sourceDefinitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8 dockerRepository: airbyte/source-google-search-console - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console icon: googlesearchconsole.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index f51d2624eb6b..6d77f9af2a46 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4521,7 +4521,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-google-search-console:0.1.17" +- dockerImage: "airbyte/source-google-search-console:0.1.18" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-search-console" connectionSpecification: