🎉 square: added oauth support

.github/workflows/publish-command.yml

@@ -151,6 +151,7 @@ jobs:
           SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }}
           SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }}
           SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
+          SOURCE_SQUARE_CREDS_OAUTH: ${{ secrets.SOURCE_SQUARE_CREDS_OAUTH }}
           SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }}
           SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
           SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}

.github/workflows/test-command.yml

@@ -146,6 +146,7 @@ jobs:
           SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }}
           SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }}
           SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
+          SOURCE_SQUARE_CREDS_OAUTH: ${{ secrets.SOURCE_SQUARE_CREDS_OAUTH }}
           SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }}
           SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
           SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}

airbyte-integrations/connectors/source-square/acceptance-test-config.yml

@@ -5,17 +5,23 @@ tests:
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config_oauth.json"
+      configured_catalog_path: "integration_tests/configured_catalog_oauth.json"
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config_oauth.json"
+      configured_catalog_path: "integration_tests/configured_catalog_oauth.json"

airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json

@@ -143,18 +143,6 @@
       "sync_mode": "full_refresh",
       "cursor_field": ["id"],
       "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "orders",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["id"]
-      },
-      "sync_mode": "full_refresh",
-      "cursor_field": ["id"],
-      "destination_sync_mode": "overwrite"
     }
   ]
 }

airbyte-integrations/connectors/source-square/integration_tests/configured_catalog_oauth.json

@@ -0,0 +1,40 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "locations",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": ["id"],
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "team_members",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": ["id"],
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "team_member_wages",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": ["id"],
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}

airbyte-integrations/connectors/source-square/integration_tests/invalid_config.json

@@ -1,6 +1,6 @@
 {
-  "api_key": "API_KEY",
-  "is_sandbox": true,
-  "start_date": "START_DATE",
-  "include_deleted_objects": false
+    "authorization": {"auth_type": "Apikey", "api_key": "bla"},
+    "is_sandbox": true,
+    "start_date": "2021-06-01",
+    "include_deleted_objects": false
 }

airbyte-integrations/connectors/source-square/source_square/source.py

@@ -12,7 +12,7 @@
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http import HttpStream
-from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator
 from source_square.utils import separate_items_by_count
 
 
@@ -358,16 +358,73 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
             yield {"location_ids": location}
 
 
+class Oauth2AuthenticatorSquare(Oauth2Authenticator):
+
+    def refresh_access_token(self):
+        """Handle differences in expiration attr:
+            from API: "expires_at": "2021-11-05T14:26:57Z"
+            expected: "expires_in": number of seconds
+        """
+        token, expires_at = super().refresh_access_token()
+        expires_in = pendulum.parse(expires_at) - pendulum.now()
+        return token, expires_in.seconds
+
+
 class SourceSquare(AbstractSource):
-    api_version = "2021-06-16"  # Latest Stable Release
+    api_version = "2021-09-15"  # Latest Stable Release
+
+    @staticmethod
+    def get_auth(config):
+
+        authorization = config.get("authorization", {})
+        auth_type = authorization.get("auth_type")
+        if auth_type == "Oauth":
+            scopes = [
+                "CUSTOMERS_READ",
+                "EMPLOYEES_READ",
+                "ITEMS_READ",
+                "MERCHANT_PROFILE_READ",
+                "ORDERS_READ",
+                "PAYMENTS_READ",
+                "TIMECARDS_READ",
+                # OAuth Permissions:
+                # https://developer.squareup.com/docs/oauth-api/square-permissions
+                # https://developer.squareup.com/reference/square/enums/OAuthPermission
+                # "DISPUTES_READ",
+                # "GIFTCARDS_READ",
+                # "INVENTORY_READ",
+                # "INVOICES_READ",
+                # "TIMECARDS_SETTINGS_READ",
+                # "LOYALTY_READ",
+                # "ONLINE_STORE_SITE_READ",
+                # "ONLINE_STORE_SNIPPETS_READ",
+                # "SUBSCRIPTIONS_READ",
+            ]
+
+            auth = Oauth2AuthenticatorSquare(
+                token_refresh_endpoint="https://connect.squareup.com/oauth2/token",
+                client_secret=authorization.get("client_secret"),
+                client_id=authorization.get("client_id"),
+                refresh_token=authorization.get("refresh_token"),
+                scopes=scopes,
+                expires_in_name="expires_at",
+            )
+        elif auth_type == "Apikey":
+            auth = TokenAuthenticator(token=authorization.get("api_key"))
+        else:
+            raise Exception(f"Invalid auth type: {auth_type}")
+
+        return auth
 
     def check_connection(self, logger, config) -> Tuple[bool, any]:
 
         headers = {
             "Square-Version": self.api_version,
-            "Authorization": "Bearer {}".format(config["api_key"]),
             "Content-Type": "application/json",
         }
+        auth = self.get_auth(config)
+        headers.update(auth.get_auth_header())
+
         url = "https://connect.squareup{}.com/v2/catalog/info".format("sandbox" if config["is_sandbox"] else "")
 
         try:
@@ -383,9 +440,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
 
-        auth = TokenAuthenticator(token=config["api_key"])
         args = {
-            "authenticator": auth,
+            "authenticator": self.get_auth(config),
             "is_sandbox": config["is_sandbox"],
             "api_version": self.api_version,
             "start_date": config["start_date"],

airbyte-integrations/connectors/source-square/source_square/spec.json

@@ -4,32 +4,90 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Square Source CDK Specifications",
     "type": "object",
-    "required": ["api_key", "is_sandbox"],
+    "required": ["authorization", "is_sandbox"],
     "additionalProperties": false,
     "properties": {
-      "api_key": {
-        "type": "string",
-        "description": "The API key for a Square application",
-        "airbyte_secret": true
-      },
       "is_sandbox": {
         "type": "boolean",
         "description": "Determines the sandbox (true) or production (false) API version",
         "examples": [true, false],
-        "default": true
+        "default": false
       },
       "start_date": {
         "type": "string",
         "description": "The start date to sync data. Leave blank for full sync. Format: YYYY-MM-DD.",
         "examples": ["2021-01-01"],
-        "default": "1970-01-01",
+        "default": "2021-01-01",
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
       },
       "include_deleted_objects": {
         "type": "boolean",
         "description": "In some streams there is and option to include deleted objects (Items, Categories, Discounts, Taxes)",
         "examples": [true, false],
         "default": false
+      },
+      "authorization": {
+        "type": "object",
+        "title": "Authentication Type",
+        "oneOf": [
+          {
+            "title": "Oauth authentication",
+            "type": "object",
+            "required": [
+              "auth_type",
+              "client_id",
+              "client_secret",
+              "refresh_token"
+            ],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "Oauth",
+                "enum": ["Oauth"],
+                "default": "Oauth",
+                "order": 0
+              },
+              "client_id": {
+                "title": "Client ID",
+                "type": "string",
+                "description": "The Square-issued ID of your application",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "title": "Client Secret",
+                "type": "string",
+                "description": "The Square-issued application secret for your application",
+                "airbyte_secret": true
+              },
+              "refresh_token": {
+                "title": "Refresh Token",
+                "type": "string",
+                "description": "A refresh token generated using the above client ID and secret",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "type": "object",
+            "title": "API Key",
+            "required": ["auth_type", "api_key"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "Apikey",
+                "enum": ["Apikey"],
+                "default": "Apikey",
+                "order": 1
+              },
+              "api_key": {
+                "title": "API key token",
+                "type": "string",
+                "description": "The API key for a Square application",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
       }
     }
   }

airbyte-integrations/connectors/source-square/unit_tests/connection_test.py

@@ -8,5 +8,12 @@
 
 def test_source_wrong_credentials():
     source = SourceSquare()
-    status, error = source.check_connection(logger=AirbyteLogger(), config={"api_key": "wrong.api.key", "is_sandbox": True})
+    config = {
+        "authorization": {"auth_type": "Apikey", "api_key": "bla"},
+        "is_sandbox": True,
+        "start_date": "2021-06-01",
+        "include_deleted_objects": False,
+    }
+    status, error = source.check_connection(logger=AirbyteLogger(), config=config)
     assert not status
+

tools/bin/ci_credentials.sh

@@ -122,6 +122,7 @@ write_standard_creds source-smartsheets "$SMARTSHEETS_TEST_CREDS"
 write_standard_creds source-snapchat-marketing "$SOURCE_SNAPCHAT_MARKETING_CREDS"
 write_standard_creds source-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "config.json"
 write_standard_creds source-square "$SOURCE_SQUARE_CREDS"
+write_standard_creds source-square "$SOURCE_SQUARE_CREDS_OAUTH" "config_oauth.json"
 write_standard_creds source-stripe "$SOURCE_STRIPE_CREDS"
 write_standard_creds source-stripe "$STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS" "connected_account_config.json"
 write_standard_creds source-surveymonkey "$SURVEYMONKEY_TEST_CREDS"