Source SmartSheets: fix oauth2 implementation (airbytehq#23237)

* Source SmartSheets: fix oauth2 implementation * Source SmartSheets: update docs * Source SmartSheets: fix tests * Source SmartSheets: fix typo; bump version * Source Smartsheets: update spec * auto-bump connector version * Automated Change * Source Smartsheets: typo --------- Co-authored-by: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com> Co-authored-by: artem1205 <artem1205@users.noreply.github.com>
danielduckworth · Mar 13, 2023 · 2103042 · 2103042
1 parent 69c8c50
commit 2103042
Show file tree

Hide file tree

Showing 10 changed files with 197 additions and 38 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1777,7 +1777,7 @@
 - name: Smartsheets
   sourceDefinitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc
   dockerRepository: airbyte/source-smartsheets
-  dockerImageTag: 0.1.14
+  dockerImageTag: 1.0.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/smartsheets
   icon: smartsheet.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -13882,29 +13882,73 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-smartsheets:0.1.14"
+- dockerImage: "airbyte/source-smartsheets:1.0.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/smartsheets"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "Smartsheets Source Spec"
       type: "object"
       required:
-      - "access_token"
+      - "credentials"
       - "spreadsheet_id"
       additionalProperties: true
       properties:
-        access_token:
-          title: "Access Token"
-          description: "The access token to use for accessing your data from Smartsheets.\
-            \ This access token must be generated by a user with at least read access\
-            \ to the data you'd like to replicate. Generate an access token in the\
-            \ Smartsheets main menu by clicking Account > Apps & Integrations > API\
-            \ Access. See the <a href=\"https://docs.airbyte.com/integrations/sources/amplitude#setup-guide\"\
-            >setup guide</a> for information on how to obtain this token."
-          type: "string"
+        credentials:
+          title: "Authorization Method"
+          type: "object"
           order: 0
-          airbyte_secret: true
+          oneOf:
+          - type: "object"
+            title: "OAuth2.0"
+            required:
+            - "client_id"
+            - "client_secret"
+            - "refresh_token"
+            - "access_token"
+            - "token_expiry_date"
+            properties:
+              auth_type:
+                type: "string"
+                const: "oauth2.0"
+              client_id:
+                type: "string"
+                description: "The API ID of the SmartSheets developer application."
+                airbyte_secret: true
+              client_secret:
+                type: "string"
+                description: "The API Secret the SmartSheets developer application."
+                airbyte_secret: true
+              access_token:
+                type: "string"
+                description: "Access Token for making authenticated requests."
+                airbyte_secret: true
+              token_expiry_date:
+                type: "string"
+                description: "The date-time when the access token should be refreshed."
+                format: "date-time"
+              refresh_token:
+                type: "string"
+                description: "The key to refresh the expired access_token."
+                airbyte_secret: true
+          - title: "API Access Token"
+            type: "object"
+            required:
+            - "access_token"
+            properties:
+              auth_type:
+                type: "string"
+                const: "access_token"
+              access_token:
+                type: "string"
+                title: "Access Token"
+                description: "The access token to use for accessing your data from\
+                  \ Smartsheets. This access token must be generated by a user with\
+                  \ at least read access to the data you'd like to replicate. Generate\
+                  \ an access token in the Smartsheets main menu by clicking Account\
+                  \ > Apps & Integrations > API Access. See the <a href=\"https://docs.airbyte.com/integrations/sources/smartsheets/#setup-guide\"\
+                  >setup guide</a> for information on how to obtain this token."
+                airbyte_secret: true
         spreadsheet_id:
           title: "Sheet ID"
           description: "The spreadsheet ID. Find it by opening the spreadsheet then\
@@ -13928,17 +13972,30 @@
     supported_destination_sync_modes: []
     advanced_auth:
       auth_flow_type: "oauth2.0"
-      predicate_key: []
-      predicate_value: ""
+      predicate_key:
+      - "credentials"
+      - "auth_type"
+      predicate_value: "oauth2.0"
       oauth_config_specification:
         complete_oauth_output_specification:
           type: "object"
-          additionalProperties: false
           properties:
             access_token:
               type: "string"
               path_in_connector_config:
+              - "credentials"
               - "access_token"
+            refresh_token:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "refresh_token"
+            token_expiry_date:
+              type: "string"
+              format: "date-time"
+              path_in_connector_config:
+              - "credentials"
+              - "token_expiry_date"
         complete_oauth_server_input_specification:
           type: "object"
           additionalProperties: false
@@ -13949,8 +14006,17 @@
               type: "string"
         complete_oauth_server_output_specification:
           type: "object"
-          additionalProperties: false
-          properties: {}
+          properties:
+            client_id:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_id"
+            client_secret:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_secret"
 - dockerImage: "airbyte/source-snapchat-marketing:0.1.15"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/snapchat-marketing"

diff --git a/airbyte-integrations/connectors/source-smartsheets/Dockerfile b/airbyte-integrations/connectors/source-smartsheets/Dockerfile
@@ -14,5 +14,5 @@ COPY $CODE_PATH ./$CODE_PATH
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.14
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-smartsheets
diff --git a/airbyte-integrations/connectors/source-smartsheets/README.md b/airbyte-integrations/connectors/source-smartsheets/README.md
@@ -1,6 +1,6 @@
 # Smartsheets Source 
 
-This is the repository for the Smartsheets source connector, written in Python. 
+This is the repository for the Smartsheets source connector, written in Python.
 For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/smartsheets).
 
 ### Author

diff --git a/airbyte-integrations/connectors/source-smartsheets/acceptance-test-config.yml b/airbyte-integrations/connectors/source-smartsheets/acceptance-test-config.yml
@@ -2,9 +2,13 @@ connector_image: airbyte/source-smartsheets:dev
 tests:
   spec:
     - spec_path: "source_smartsheets/spec.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.14"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:

diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py
@@ -2,23 +2,41 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+import hashlib
 import logging
 from functools import cached_property
 from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
 
 import smartsheet
+from airbyte_cdk.sources.streams.http.requests_native_auth import SingleUseRefreshTokenOauth2Authenticator
 
 
 class SmartSheetAPIWrapper:
     def __init__(self, config: Mapping[str, Any]):
         self._spreadsheet_id = config["spreadsheet_id"]
-        self._access_token = config["access_token"]
-        api_client = smartsheet.Smartsheet(self._access_token)
-        api_client.errors_as_exceptions(True)
+        self._config = config
+        self.api_client = smartsheet.Smartsheet(self.get_access_token(config))
+        self.api_client.errors_as_exceptions(True)
         # each call to `Sheets` makes a new instance, so we save it here to make no more new objects
-        self._get_sheet = api_client.Sheets.get_sheet
+        self._get_sheet = self.api_client.Sheets.get_sheet
         self._data = None
 
+    def get_token_hash(self, config: Mapping[str, Any]):
+        credentials = config.get("credentials")
+        return {"hash": hashlib.sha256(f"{credentials.get('client_secret')}|{credentials.get('refresh_token')}".encode()).hexdigest()}
+
+    def get_access_token(self, config: Mapping[str, Any]):
+        credentials = config.get("credentials")
+        if config.get("credentials", {}).get("auth_type") == "oauth2.0":
+            authenticator = SingleUseRefreshTokenOauth2Authenticator(
+                config, token_refresh_endpoint="https://api.smartsheet.com/2.0/token", refresh_request_body=self.get_token_hash(config)
+            )
+            return authenticator.get_access_token()
+
+        else:
+            access_token = credentials.get("access_token")
+        return access_token
+
     def _fetch_sheet(self, from_dt: Optional[str] = None) -> None:
         kwargs = {"rows_modified_since": from_dt}
         if not from_dt:
@@ -43,6 +61,7 @@ def _construct_record(self, row: smartsheet.models.Row) -> Dict[str, str]:
     @property
     def data(self) -> smartsheet.models.Row:
         if not self._data:
+            self.api_client._access_token = self.get_access_token(self._config)
             self._fetch_sheet()
         return self._data
 

diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/spec.json b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/spec.json
@@ -4,15 +4,68 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Smartsheets Source Spec",
     "type": "object",
-    "required": ["access_token", "spreadsheet_id"],
+    "required": ["credentials", "spreadsheet_id"],
     "additionalProperties": true,
     "properties": {
-      "access_token": {
-        "title": "Access Token",
-        "description": "The access token to use for accessing your data from Smartsheets. This access token must be generated by a user with at least read access to the data you'd like to replicate. Generate an access token in the Smartsheets main menu by clicking Account > Apps & Integrations > API Access. See the <a href=\"https://docs.airbyte.com/integrations/sources/amplitude#setup-guide\">setup guide</a> for information on how to obtain this token.",
-        "type": "string",
+      "credentials": {
+        "title": "Authorization Method",
+        "type": "object",
         "order": 0,
-        "airbyte_secret": true
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "OAuth2.0",
+            "required": ["client_id", "client_secret", "refresh_token", "access_token", "token_expiry_date"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "oauth2.0"
+              },
+              "client_id": {
+                "type": "string",
+                "description": "The API ID of the SmartSheets developer application.",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "type": "string",
+                "description": "The API Secret the SmartSheets developer application.",
+                "airbyte_secret": true
+              },
+              "access_token": {
+                "type": "string",
+                "description": "Access Token for making authenticated requests.",
+                "airbyte_secret": true
+              },
+              "token_expiry_date": {
+                "type": "string",
+                "description": "The date-time when the access token should be refreshed.",
+                "format": "date-time"
+              },
+              "refresh_token": {
+                "type": "string",
+                "description": "The key to refresh the expired access_token.",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "title": "API Access Token",
+            "type": "object",
+            "required": ["access_token"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "access_token"
+              },
+              "access_token": {
+                "type": "string",
+                "title": "Access Token",
+                "description": "The access token to use for accessing your data from Smartsheets. This access token must be generated by a user with at least read access to the data you'd like to replicate. Generate an access token in the Smartsheets main menu by clicking Account > Apps & Integrations > API Access. See the <a href=\"https://docs.airbyte.com/integrations/sources/smartsheets/#setup-guide\">setup guide</a> for information on how to obtain this token.",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
       },
       "spreadsheet_id": {
         "title": "Sheet ID",
@@ -34,16 +87,24 @@
   },
   "advanced_auth": {
     "auth_flow_type": "oauth2.0",
-    "predicate_key": [],
-    "predicate_value": "",
+    "predicate_key": ["credentials", "auth_type"],
+    "predicate_value": "oauth2.0",
     "oauth_config_specification": {
       "complete_oauth_output_specification": {
         "type": "object",
-        "additionalProperties": false,
         "properties": {
           "access_token": {
             "type": "string",
-            "path_in_connector_config": ["access_token"]
+            "path_in_connector_config": ["credentials", "access_token"]
+          },
+          "refresh_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "refresh_token"]
+          },
+          "token_expiry_date": {
+            "type": "string",
+            "format": "date-time",
+            "path_in_connector_config": ["credentials", "token_expiry_date"]
           }
         }
       },
@@ -61,8 +122,16 @@
       },
       "complete_oauth_server_output_specification": {
         "type": "object",
-        "additionalProperties": false,
-        "properties": {}
+        "properties": {
+          "client_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_id"]
+          },
+          "client_secret": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_secret"]
+          }
+        }
       }
     }
   }

diff --git a/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py b/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py
@@ -20,7 +20,7 @@ def response_mock():
 
 @pytest.fixture
 def config():
-    return {"spreadsheet_id": "id", "access_token": "token"}
+    return {"spreadsheet_id": "id", "credentials": {"access_token": "token"}}
 
 
 @pytest.fixture

diff --git a/connectors.md b/connectors.md
@@ -214,7 +214,7 @@
 | **Slack** | <img alt="Slack icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/slack.svg" height="30" height="30"/> | Source | airbyte/source-slack:0.1.23 | generally_available | [link](https://docs.airbyte.com/integrations/sources/slack) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-slack) | <small>`c2281cee-86f9-4a86-bb48-d23286b4c7bd`</small> |
 | **Smaily** | <img alt="Smaily icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/smaily.svg" height="30" height="30"/> | Source | airbyte/source-smaily:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/smaily) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-smaily) | <small>`781f8b1d-4e20-4842-a2c3-cd9b119d65fa`</small> |
 | **SmartEngage** | <img alt="SmartEngage icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/smartengage.svg" height="30" height="30"/> | Source | airbyte/source-smartengage:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/smartengage) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-smartengage) | <small>`21cc4a17-a011-4485-8a3e-e2341a91ab9f`</small> |
-| **Smartsheets** | <img alt="Smartsheets icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/smartsheet.svg" height="30" height="30"/> | Source | airbyte/source-smartsheets:0.1.14 | beta | [link](https://docs.airbyte.com/integrations/sources/smartsheets) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-smartsheets) | <small>`374ebc65-6636-4ea0-925c-7d35999a8ffc`</small> |
+| **Smartsheets** | <img alt="Smartsheets icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/smartsheet.svg" height="30" height="30"/> | Source | airbyte/source-smartsheets:1.0.0 | beta | [link](https://docs.airbyte.com/integrations/sources/smartsheets) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-smartsheets) | <small>`374ebc65-6636-4ea0-925c-7d35999a8ffc`</small> |
 | **Snapchat Marketing** | <img alt="Snapchat Marketing icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/snapchat.svg" height="30" height="30"/> | Source | airbyte/source-snapchat-marketing:0.1.15 | generally_available | [link](https://docs.airbyte.com/integrations/sources/snapchat-marketing) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-snapchat-marketing) | <small>`200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b`</small> |
 | **Snowflake** | <img alt="Snowflake icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/snowflake.svg" height="30" height="30"/> | Source | airbyte/source-snowflake:0.1.30 | alpha | [link](https://docs.airbyte.com/integrations/sources/snowflake) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-snowflake) | <small>`e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2`</small> |
 | **Sonar Cloud** | <img alt="Sonar Cloud icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/sonarcloud.svg" height="30" height="30"/> | Source | airbyte/source-sonar-cloud:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/sonar-cloud) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sonar-cloud) | <small>`3ab1d7d0-1577-4ab9-bcc4-1ff6a4c2c9f2`</small> |

diff --git a/docs/integrations/sources/smartsheets.md b/docs/integrations/sources/smartsheets.md
@@ -88,6 +88,7 @@ The remaining column datatypes supported by Smartsheets are more complex types (
 
 | Version | Date       | Pull Request                                             | Subject                                                   |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------|
+| 1.0.0   | 2023-02-19 | [23237](https://github.com/airbytehq/airbyte/pull/23237) | Fix OAuth2.0 token refresh                                |
 | 0.1.14  | 2023-02-07 | [22419](https://github.com/airbytehq/airbyte/pull/22419) | OAuth2.0 support - enabled; add allowed hosts             |
 | 0.1.13  | 2022-12-02 | [20017](https://github.com/airbytehq/airbyte/pull/20017) | OAuth2.0 support - disabled                               |
 | 0.1.12  | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy                          |