MINOR - update docs & added unit test for SAS Connector

ingestion/src/metadata/ingestion/source/database/sas/client.py

@@ -31,7 +31,7 @@ class SASClient:
 
     def __init__(self, config: SASConnection):
         self.config: SASConnection = config
-        self.auth_token = get_token(
+        self.auth_token = self.get_token(
             config.serverHost, config.username, config.password.get_secret_value()
         )
         client_config: ClientConfig = ClientConfig(
@@ -99,8 +99,7 @@ def list_assets(self, assets):
         )
         endpoint = (
             f"catalog/search?indices={assets}&q="
-            # f"{asset_filter if str(asset_filter) != 'None' else '*'}"
-            f"{asset_filter if str(asset_filter) != 'None' else '*'}&limit=10"  # TODO: MAKE THE CHANGE
+            f"{asset_filter if str(asset_filter) != 'None' else '*'}"
         )
         headers = {"Accept-Item": "application/vnd.sas.metadata.instance.entity+json"}
         response = self.client._request("GET", path=endpoint, headers=headers)
@@ -167,18 +166,19 @@ def get_instances_with_param(self, data):
     def get_auth_token(self):
         return self.auth_token, 0
 
-
-def get_token(base_url, user, password):
-    endpoint = "/SASLogon/oauth/token"
-    payload = {"grant_type": "password", "username": user, "password": password}
-    headers = {
-        "Content-type": "application/x-www-form-urlencoded",
-        "Authorization": "Basic c2FzLmNsaTo=",
-    }
-    url = base_url + endpoint
-    response = requests.request(
-        "POST", url, headers=headers, data=payload, verify=False, timeout=10
-    )
-    text_response = response.json()
-    logger.info(f"this is user: {user}, password: {password}, text: {text_response}")
-    return response.json()["access_token"]
+    def get_token(self, base_url, user, password):
+        endpoint = "/SASLogon/oauth/token"
+        payload = {"grant_type": "password", "username": user, "password": password}
+        headers = {
+            "Content-type": "application/x-www-form-urlencoded",
+            "Authorization": "Basic c2FzLmNsaTo=",
+        }
+        url = base_url + endpoint
+        response = requests.request(
+            "POST", url, headers=headers, data=payload, verify=False, timeout=10
+        )
+        text_response = response.json()
+        logger.info(
+            f"this is user: {user}, password: {password}, text: {text_response}"
+        )
+        return response.json()["access_token"]

ingestion/src/metadata/ingestion/source/database/sas/metadata.py

@@ -72,7 +72,7 @@
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_connection
+from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.database_service import DatabaseServiceSource
 from metadata.ingestion.source.database.sas.client import SASClient
@@ -133,6 +133,13 @@ def create(cls, config_dict: dict, metadata: OpenMetadata):
         return cls(config, metadata)
 
     def _iter(self) -> Iterable[Either[Entity]]:
+        # For debug purpose (when ingesting locally)
+        # yield Either(
+        #     right=self.metadata.get_create_service_from_source(
+        #         entity=DatabaseService, config=self.config
+        #     )
+        # )
+
         # create tables from sas dataSets
         if self.sas_client.enable_datatables:
             for table in self.sas_client.list_assets("datasets"):
@@ -500,10 +507,6 @@ def create_table_entity(self, table) -> Iterable[Either[CreateTableRequest]]:
                     if attr in custom_attributes
                 }
 
-                # create table request
-                logger.info(
-                    f"schema: {table['id']}, {self.db_service_name}, {self.db_name}, {self.db_schema_name}"
-                )
                 table_request = CreateTableRequest(
                     name=table_name,
                     sourceUrl=table_url,
@@ -906,3 +909,7 @@ def yield_procedure_lineage_and_queries(
 
     def close(self) -> None:
         pass
+
+    def test_connection(self) -> None:
+        test_connection_fn = get_test_connection_fn(self.service_connection)
+        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)

ingestion/tests/unit/resources/datasets/sas_dataset_search.json

@@ -0,0 +1,43 @@
+[
+  {
+    "id": "0063116c-577c-0f44-8116-3924506c8f4a",
+    "version": 1,
+    "name": "WATER_CLUSTER.sashdat",
+    "type": "casTable",
+    "owner": "sas",
+    "created": "2023-12-20T08:28:18.000Z",
+    "modified": "2023-12-20T08:28:18.500Z",
+    "registeredType": "dataTable",
+    "displayType": "CAS",
+    "score": 24.40445,
+    "attributes": {
+      "analysisTimeStamp": "2023-12-20T20:52:01.453Z",
+      "columnCount": 21,
+      "completenessPercent": 95,
+      "dataSize": 10091520,
+      "extension": "sashdat",
+      "informationPrivacy": "candidate",
+      "library": "Samples",
+      "region": "",
+      "reviewStatus": "none",
+      "rowCount": 46720,
+      "sourceSystem": "CAS"
+    },
+    "links": [
+      {
+        "method": "GET",
+        "rel": "self",
+        "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a",
+        "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a",
+        "type": "application/vnd.sas.metadata.instance"
+      },
+      {
+        "method": "GET",
+        "rel": "resource",
+        "href": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER?ext=sashdat",
+        "uri": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER?ext=sashdat",
+        "type": "application/json"
+      }
+    ]
+  }
+]