Source: Google Search Console: validate custom_reports.dimension (air…

…bytehq#18568) Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
jhammarstedt · Oct 31, 2022 · 3d78a96 · 3d78a96
1 parent 8bc8089
commit 3d78a96
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 12 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -460,7 +460,7 @@
 - name: Google Search Console
   sourceDefinitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8
   dockerRepository: airbyte/source-google-search-console
-  dockerImageTag: 0.1.17
+  dockerImageTag: 0.1.18
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console
   icon: googlesearchconsole.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4521,7 +4521,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-google-search-console:0.1.17"
+- dockerImage: "airbyte/source-google-search-console:0.1.18"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/google-search-console"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.17
+LABEL io.airbyte.version=0.1.18
 LABEL io.airbyte.name=airbyte/source-google-search-console
diff --git a/...tegrations/connectors/source-google-search-console/source_google_search_console/source.py b/...tegrations/connectors/source-google-search-console/source_google_search_console/source.py
@@ -63,6 +63,10 @@ def _validate_and_transform(self, config: Mapping[str, Any]):
             except ValueError:
                 raise Exception("custom_reports is not valid JSON")
             jsonschema.validate(config["custom_reports"], custom_reports_schema)
+            for report in config["custom_reports"]:
+                for dimension in report["dimensions"]:
+                    if dimension not in SearchAnalyticsByCustomDimensions.dimension_to_property_schema_map:
+                        raise Exception(f"dimension: '{dimension}' not found")
 
         pendulum.parse(config["start_date"])
         end_date = config.get("end_date")

diff --git a/...egrations/connectors/source-google-search-console/source_google_search_console/streams.py b/...egrations/connectors/source-google-search-console/source_google_search_console/streams.py
@@ -298,14 +298,22 @@ class SearchAnalyticsAllFields(SearchAnalytics):
 
 
 class SearchAnalyticsByCustomDimensions(SearchAnalytics):
+    dimension_to_property_schema_map = {
+        "country": [{"country": {"type": ["null", "string"]}}],
+        "date": [],
+        "device": [{"device": {"type": ["null", "string"]}}],
+        "page": [{"page": {"type": ["null", "string"]}}],
+        "query": [{"query": {"type": ["null", "string"]}}],
+    }
+
     def __init__(self, dimensions: List[str], *args, **kwargs):
         super(SearchAnalyticsByCustomDimensions, self).__init__(*args, **kwargs)
         self.dimensions = dimensions
 
     def get_json_schema(self) -> Mapping[str, Any]:
         try:
             return super(SearchAnalyticsByCustomDimensions, self).get_json_schema()
-        except IOError:
+        except FileNotFoundError:
             schema: Mapping[str, Any] = {
                 "$schema": "http://json-schema.org/draft-07/schema#",
                 "type": ["null", "object"],
@@ -327,16 +335,9 @@ def get_json_schema(self) -> Mapping[str, Any]:
             return schema
 
     def dimension_to_property_schema(self) -> dict:
-        dimension_to_property_schema_map = {
-            "country": [{"country": {"type": ["null", "string"]}}],
-            "date": [],
-            "device": [{"device": {"type": ["null", "string"]}}],
-            "page": [{"page": {"type": ["null", "string"]}}],
-            "query": [{"query": {"type": ["null", "string"]}}],
-        }
         properties = {}
         for dimension in sorted(self.dimensions):
-            fields = dimension_to_property_schema_map[dimension]
+            fields = self.dimension_to_property_schema_map[dimension]
             for field in fields:
                 properties = {**properties, **field}
         return properties
diff --git a/docs/integrations/sources/google-search-console.md b/docs/integrations/sources/google-search-console.md
@@ -128,6 +128,7 @@ This connector attempts to back off gracefully when it hits Reports API's rate l
 
 | Version  | Date       | Pull Request                                                                                                  | Subject                                                     |
 | :------- | :--------- | :------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------- |
+| `0.1.18` | 2022-10-27 | [18568](https://github.com/airbytehq/airbyte/pull/18568)                                                      | Improved config validation: custom_reports.dimension        |
 | `0.1.17` | 2022-10-08 | [17751](https://github.com/airbytehq/airbyte/pull/17751)                                                      | Improved config validation: start_date, end_date, site_urls |
 | `0.1.16` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304)                                                      | Migrate to per-stream state.                                |
 | `0.1.15` | 2022-09-16 | [16819](https://github.com/airbytehq/airbyte/pull/16819)                                                      | Check available site urls to avoid 403 error on sync        |