🐛Source Looker: Fix schema transformation issue (#20182)

* Fix schema transformation issue

* Updated PR number

* Added unittest

* Add a test case that uses the recursion

* Unhide from cloud

* Bumed seed version

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -890,7 +890,7 @@
 - name: Looker
   sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c
   dockerRepository: airbyte/source-looker
-  dockerImageTag: 0.2.7
+  dockerImageTag: 0.2.8
   documentationUrl: https://docs.airbyte.com/integrations/sources/looker
   icon: looker.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -7610,7 +7610,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-looker:0.2.7"
+- dockerImage: "airbyte/source-looker:0.2.8"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/looker"
     connectionSpecification:
@@ -7621,7 +7621,7 @@
       - "domain"
       - "client_id"
       - "client_secret"
-      additionalProperties: false
+      additionalProperties: true
       properties:
         domain:
           type: "string"
@@ -7642,6 +7642,7 @@
           title: "Client Secret"
           type: "string"
           description: "The Client Secret is second part of an API3 key."
+          airbyte_secret: true
         run_look_ids:
           title: "Look IDs to Run"
           type: "array"

airbyte-integrations/connectors/source-looker/Dockerfile

@@ -33,5 +33,5 @@ COPY source_looker ./source_looker
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.7
+LABEL io.airbyte.version=0.2.8
 LABEL io.airbyte.name=airbyte/source-looker

airbyte-integrations/connectors/source-looker/README.md

@@ -83,7 +83,7 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat
 Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
 First install test dependencies into your virtual environment:
 ```
-pip install .[tests]
+pip install .'[tests]'
 ```
 ### Unit Tests
 To run unit tests locally, from the connector directory run:

airbyte-integrations/connectors/source-looker/source_looker/spec.json

@@ -5,7 +5,7 @@
     "title": "Looker Spec",
     "type": "object",
     "required": ["domain", "client_id", "client_secret"],
-    "additionalProperties": false,
+    "additionalProperties": true,
     "properties": {
       "domain": {
         "type": "string",
@@ -25,7 +25,8 @@
       "client_secret": {
         "title": "Client Secret",
         "type": "string",
-        "description": "The Client Secret is second part of an API3 key."
+        "description": "The Client Secret is second part of an API3 key.",
+        "airbyte_secret": true
       },
       "run_look_ids": {
         "title": "Look IDs to Run",

airbyte-integrations/connectors/source-looker/source_looker/streams.py

@@ -251,10 +251,40 @@ def get_parent_endpoints(self) -> List[SwaggerParser.Endpoint]:
     def name(self) -> str:
         return self._name
 
+    @classmethod
+    def format_null_in_schema(cls, schema: Mapping[str, Any]):
+        """Add 'null' to schema type field
+
+        Output:
+            {
+            ...
+            'type': ['null', 'object']
+            'properties':   {
+                'field':{
+                    'type': ['null', 'number']
+                        }
+                }
+            ...
+            }
+
+        """
+
+        for key, value_schema in schema.items():
+            if isinstance(value_schema, dict):
+                schema_type = value_schema.get("type")
+
+                if isinstance(schema_type, str):
+                    value_schema["type"] = ["null", schema_type]
+
+                schema[key] = cls.format_null_in_schema(value_schema)
+
+        return schema
+
     def get_json_schema(self) -> Mapping[str, Any]:
         # Overrides default logic. All schema should be generated dynamically.
         schema = self.endpoint.schema.get("items") or self.endpoint.schema
-        return {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": schema["properties"]}
+        schema = self.format_null_in_schema(schema["properties"])
+        return {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": schema}
 
     def path(self, stream_slice: Mapping[str, Any], **kwargs: Any) -> str:
         stream_slice = stream_slice or {}
@@ -473,7 +503,7 @@ def get_json_schema(self) -> Mapping[str, Any]:
                 "title": look_info["title"],
                 "properties": look_properties,
                 "type": ["null", "object"],
-                "additionalProperties": False,
+                "additionalProperties": True,
             }
         # raise LookerException(properties)
         return {

airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py

@@ -2,7 +2,11 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from source_looker.streams import LookerStream
 
-def test_example_method() -> None:
-    assert True
-    return
+
+def test_format_null_in_schema():
+    schema = {'type': ['null', 'object'], 'properties': {'field': {'type': 'object', 'properties': {'field': {'type': 'number'}}}}}
+    output = LookerStream.format_null_in_schema(schema)
+    expected = {'type': ['null', 'object'], 'properties': {'field': {'type': ['null', 'object'], 'properties': {'field': {'type': ['null', 'number']}}}}}
+    assert output == expected

airbyte-webapp/src/core/domain/connector/constants.ts

@@ -31,7 +31,6 @@ export const getExcludedConnectorIds = (workspaceId?: string): string[] =>
         ConnectorIds.Destinations.RabbitMq, // hide RabbitMQ Destination https://github.com/airbytehq/airbyte/issues/16315
         ConnectorIds.Destinations.AmazonSqs, // hide Amazon SQS Destination https://github.com/airbytehq/airbyte/issues/16316
         ConnectorIds.Sources.AmazonSellerPartner, // hide Amazon Seller Partner Source https://github.com/airbytehq/airbyte/issues/14734
-        ConnectorIds.Sources.Looker, // hide Looker Source https://github.com/airbytehq/alpha-beta-issues/issues/39
         ...(workspaceId !== "54135667-ce73-4820-a93c-29fe1510d348" // Shopify workspace for review
           ? [ConnectorIds.Sources.Shopify] // Shopify
           : []),

docs/integrations/sources/looker.md

@@ -78,14 +78,15 @@ Please read the "API3 Key" section in [Looker's information for users docs](http
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
-| 0.2.7 | 2022-01-24 | [\#9609](https://github.com/airbytehq/airbyte/pull/9609) | Migrate to native CDK and fixing of intergration tests |
-| 0.2.6 | 2021-12-07 | [\#8578](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions |
-| 0.2.5 | 2021-10-27 | [\#7284](https://github.com/airbytehq/airbyte/pull/7284) | Migrate Looker source to CDK structure, add SAT testing. |
-| 0.2.4 | 2021-06-25 | [\#3911](https://github.com/airbytehq/airbyte/pull/3911) | Added `run_look` endpoint. |
-| 0.2.3 | 2021-06-22 | [\#3587](https://github.com/airbytehq/airbyte/pull/3587) | Added support for self-hosted instances. |
-| 0.2.2 | 2021-06-09 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` for kubernetes support. |
-| 0.2.1 | 2021-04-02 | [\#2726](https://github.com/airbytehq/airbyte/pull/2726) | Fixed connector base versioning. |
-| 0.2.0 | 2021-03-09 | [\#2238](https://github.com/airbytehq/airbyte/pull/2238) | Allowed future / unknown properties in the protocol. |
-| 0.1.1 | 2021-01-27 | [\#1857](https://github.com/airbytehq/airbyte/pull/1857) | Fix failed CI tests. |
-| 0.1.0 | 2020-12-24 | [\#1441](https://github.com/airbytehq/airbyte/pull/1441) | Added looker connector. |
+| 0.2.8 | 2022-12-07 | [20182](https://github.com/airbytehq/airbyte/pull/20182) | Fix schema transformation issue |
+| 0.2.7 | 2022-01-24 | [9609](https://github.com/airbytehq/airbyte/pull/9609) | Migrate to native CDK and fixing of intergration tests. |
+| 0.2.6 | 2021-12-07 | [8578](https://github.com/airbytehq/airbyte/pull/8578) | Update titles and descriptions. |
+| 0.2.5 | 2021-10-27 | [7284](https://github.com/airbytehq/airbyte/pull/7284) | Migrate Looker source to CDK structure, add SAT testing. |
+| 0.2.4 | 2021-06-25 | [3911](https://github.com/airbytehq/airbyte/pull/3911) | Add `run_look` endpoint. |
+| 0.2.3 | 2021-06-22 | [3587](https://github.com/airbytehq/airbyte/pull/3587) | Add support for self-hosted instances. |
+| 0.2.2 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for kubernetes support. |
+| 0.2.1 | 2021-04-02 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix connector base versioning. |
+| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Allow future / unknown properties in the protocol. |
+| 0.1.1 | 2021-01-27 | [1857](https://github.com/airbytehq/airbyte/pull/1857) | Fix failed CI tests. |
+| 0.1.0 | 2020-12-24 | [1441](https://github.com/airbytehq/airbyte/pull/1441) | Add looker connector. |