diff --git a/ingestion/src/metadata/ingestion/source/database/sas/client.py b/ingestion/src/metadata/ingestion/source/database/sas/client.py index f61693aad578..7ccc832a46b5 100644 --- a/ingestion/src/metadata/ingestion/source/database/sas/client.py +++ b/ingestion/src/metadata/ingestion/source/database/sas/client.py @@ -31,7 +31,7 @@ class SASClient: def __init__(self, config: SASConnection): self.config: SASConnection = config - self.auth_token = get_token( + self.auth_token = self.get_token( config.serverHost, config.username, config.password.get_secret_value() ) client_config: ClientConfig = ClientConfig( @@ -99,8 +99,7 @@ def list_assets(self, assets): ) endpoint = ( f"catalog/search?indices={assets}&q=" - # f"{asset_filter if str(asset_filter) != 'None' else '*'}" - f"{asset_filter if str(asset_filter) != 'None' else '*'}&limit=10" # TODO: MAKE THE CHANGE + f"{asset_filter if str(asset_filter) != 'None' else '*'}" ) headers = {"Accept-Item": "application/vnd.sas.metadata.instance.entity+json"} response = self.client._request("GET", path=endpoint, headers=headers) @@ -167,18 +166,19 @@ def get_instances_with_param(self, data): def get_auth_token(self): return self.auth_token, 0 - -def get_token(base_url, user, password): - endpoint = "/SASLogon/oauth/token" - payload = {"grant_type": "password", "username": user, "password": password} - headers = { - "Content-type": "application/x-www-form-urlencoded", - "Authorization": "Basic c2FzLmNsaTo=", - } - url = base_url + endpoint - response = requests.request( - "POST", url, headers=headers, data=payload, verify=False, timeout=10 - ) - text_response = response.json() - logger.info(f"this is user: {user}, password: {password}, text: {text_response}") - return response.json()["access_token"] + def get_token(self, base_url, user, password): + endpoint = "/SASLogon/oauth/token" + payload = {"grant_type": "password", "username": user, "password": password} + headers = { + "Content-type": "application/x-www-form-urlencoded", + "Authorization": "Basic c2FzLmNsaTo=", + } + url = base_url + endpoint + response = requests.request( + "POST", url, headers=headers, data=payload, verify=False, timeout=10 + ) + text_response = response.json() + logger.info( + f"this is user: {user}, password: {password}, text: {text_response}" + ) + return response.json()["access_token"] diff --git a/ingestion/src/metadata/ingestion/source/database/sas/metadata.py b/ingestion/src/metadata/ingestion/source/database/sas/metadata.py index cda4f6b75c54..7440ecdcc41b 100644 --- a/ingestion/src/metadata/ingestion/source/database/sas/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/sas/metadata.py @@ -72,7 +72,7 @@ from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.ingestion.source.connections import get_connection +from metadata.ingestion.source.connections import get_connection, get_test_connection_fn from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser from metadata.ingestion.source.database.database_service import DatabaseServiceSource from metadata.ingestion.source.database.sas.client import SASClient @@ -133,6 +133,13 @@ def create(cls, config_dict: dict, metadata: OpenMetadata): return cls(config, metadata) def _iter(self) -> Iterable[Either[Entity]]: + # For debug purpose (when ingesting locally) + # yield Either( + # right=self.metadata.get_create_service_from_source( + # entity=DatabaseService, config=self.config + # ) + # ) + # create tables from sas dataSets if self.sas_client.enable_datatables: for table in self.sas_client.list_assets("datasets"): @@ -500,10 +507,6 @@ def create_table_entity(self, table) -> Iterable[Either[CreateTableRequest]]: if attr in custom_attributes } - # create table request - logger.info( - f"schema: {table['id']}, {self.db_service_name}, {self.db_name}, {self.db_schema_name}" - ) table_request = CreateTableRequest( name=table_name, sourceUrl=table_url, @@ -906,3 +909,7 @@ def yield_procedure_lineage_and_queries( def close(self) -> None: pass + + def test_connection(self) -> None: + test_connection_fn = get_test_connection_fn(self.service_connection) + test_connection_fn(self.metadata, self.connection_obj, self.service_connection) diff --git a/ingestion/tests/unit/resources/datasets/sas_dataset_search.json b/ingestion/tests/unit/resources/datasets/sas_dataset_search.json new file mode 100644 index 000000000000..72eb660ec8e6 --- /dev/null +++ b/ingestion/tests/unit/resources/datasets/sas_dataset_search.json @@ -0,0 +1,43 @@ +[ + { + "id": "0063116c-577c-0f44-8116-3924506c8f4a", + "version": 1, + "name": "WATER_CLUSTER.sashdat", + "type": "casTable", + "owner": "sas", + "created": "2023-12-20T08:28:18.000Z", + "modified": "2023-12-20T08:28:18.500Z", + "registeredType": "dataTable", + "displayType": "CAS", + "score": 24.40445, + "attributes": { + "analysisTimeStamp": "2023-12-20T20:52:01.453Z", + "columnCount": 21, + "completenessPercent": 95, + "dataSize": 10091520, + "extension": "sashdat", + "informationPrivacy": "candidate", + "library": "Samples", + "region": "", + "reviewStatus": "none", + "rowCount": 46720, + "sourceSystem": "CAS" + }, + "links": [ + { + "method": "GET", + "rel": "self", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "type": "application/vnd.sas.metadata.instance" + }, + { + "method": "GET", + "rel": "resource", + "href": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER?ext=sashdat", + "uri": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER?ext=sashdat", + "type": "application/json" + } + ] + } +] \ No newline at end of file diff --git a/ingestion/tests/unit/resources/datasets/sas_dataset_view.json b/ingestion/tests/unit/resources/datasets/sas_dataset_view.json new file mode 100644 index 000000000000..8bb416770918 --- /dev/null +++ b/ingestion/tests/unit/resources/datasets/sas_dataset_view.json @@ -0,0 +1,496 @@ +{ + "version": 1, + "entities": [ + { + "creationTimeStamp": "2023-12-20T20:51:57.437Z", + "createdBy": "admin", + "modifiedTimeStamp": "2023-12-20T20:51:57.437Z", + "modifiedBy": "admin", + "id": "a278a7b0-48f2-314b-a2be-fd2753dc7465", + "links": [ + { + "method": "GET", + "rel": "self", + "href": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "uri": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "type": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "alternate", + "href": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "uri": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "type": "application/vnd.sas.summary" + }, + { + "method": "GET", + "rel": "up", + "href": "/catalog/instances", + "uri": "/catalog/instances", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance" + }, + { + "method": "PATCH", + "rel": "patch", + "href": "/catalog/updates/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "uri": "/catalog/updates/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "type": "application/json-patch", + "responseType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "PUT", + "rel": "update", + "href": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "uri": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "type": "application/vnd.sas.metadata.instance.entity", + "responseType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "DELETE", + "rel": "delete", + "href": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "uri": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465" + }, + { + "method": "GET", + "rel": "definition", + "href": "/catalog/definitions/87d48d12-53c0-473c-ab34-531817319e79", + "uri": "/catalog/definitions/87d48d12-53c0-473c-ab34-531817319e79", + "type": "application/vnd.sas.metadata.definition.entity" + }, + { + "method": "GET", + "rel": "history", + "href": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465/history", + "uri": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465/history", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "resource", + "href": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER/columns/Address", + "uri": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER/columns/Address", + "type": "application/json" + } + ], + "version": 1, + "instanceType": "entity", + "definitionId": "87d48d12-53c0-473c-ab34-531817319e79", + "name": "Address", + "type": "casColumn", + "attributes": { + "actualDataType": "String", + "analysisTimeStamp": "2023-12-20T20:52:01.453Z", + "bestChartType": "frequency", + "blankValueCount": 0, + "cardinalityCount": 63, + "casDataType": "char", + "charsMaxCount": 25, + "charsMinCount": 11, + "completenessPercent": 100, + "dataType": "char", + "hasUniqueField": false, + "informationPrivacy": "candidate", + "leastCommonValue": "1650 FEDERAL RD", + "mismatchedCount": 0, + "missingCount": 0, + "mode": "1980 OIST OAK BLVD", + "mostCommonValue": "1980 OIST OAK BLVD", + "ordinalPosition": 7, + "rawLength": 28, + "sasDataType": "char", + "sasFormat": "$CHAR", + "sasFormatFull": "$CHAR28.", + "semanticTypeId": "ADDRESS", + "semanticTypeScore": 201, + "statisticalScale": "nominal", + "uniquenessPercent": 0.1348458904 + }, + "resourceId": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER/columns/Address" + }, + { + "creationTimeStamp": "2023-12-20T20:51:57.405Z", + "createdBy": "admin", + "modifiedTimeStamp": "2023-12-20T20:51:57.405Z", + "modifiedBy": "admin", + "id": "5569fb65-c6df-484a-897e-7a087665cd6c", + "links": [ + { + "method": "GET", + "rel": "self", + "href": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "uri": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "type": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "alternate", + "href": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "uri": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "type": "application/vnd.sas.summary" + }, + { + "method": "GET", + "rel": "up", + "href": "/catalog/instances", + "uri": "/catalog/instances", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance" + }, + { + "method": "PATCH", + "rel": "patch", + "href": "/catalog/updates/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "uri": "/catalog/updates/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "type": "application/json-patch", + "responseType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "PUT", + "rel": "update", + "href": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "uri": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "type": "application/vnd.sas.metadata.instance.entity", + "responseType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "DELETE", + "rel": "delete", + "href": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "uri": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c" + }, + { + "method": "GET", + "rel": "definition", + "href": "/catalog/definitions/87d48d12-53c0-473c-ab34-531817319e79", + "uri": "/catalog/definitions/87d48d12-53c0-473c-ab34-531817319e79", + "type": "application/vnd.sas.metadata.definition.entity" + }, + { + "method": "GET", + "rel": "history", + "href": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c/history", + "uri": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c/history", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "resource", + "href": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER/columns/US%20Holiday", + "uri": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER/columns/US%20Holiday", + "type": "application/json" + } + ], + "version": 1, + "instanceType": "entity", + "definitionId": "87d48d12-53c0-473c-ab34-531817319e79", + "name": "US Holiday", + "type": "casColumn", + "attributes": { + "actualDataType": "String", + "analysisTimeStamp": "2023-12-20T20:52:01.453Z", + "bestChartType": "frequency", + "blankValueCount": 0, + "cardinalityCount": 15, + "casDataType": "char", + "charsMaxCount": 27, + "charsMinCount": 8, + "completenessPercent": 3, + "dataType": "char", + "hasUniqueField": false, + "leastCommonValue": "Independence Day", + "mismatchedCount": 0, + "missingCount": 45056, + "mode": "Veteran's Day", + "mostCommonValue": "Veteran's Day", + "ordinalPosition": 20, + "rawLength": 27, + "sasDataType": "char", + "sasFormat": "$CHAR", + "sasFormatFull": "$CHAR27.", + "semanticTypeScore": 7.12, + "statisticalScale": "nominal", + "uniquenessPercent": 0.0321061644 + }, + "resourceId": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER/columns/US%20Holiday" + }, + { + "creationTimeStamp": "2023-12-20T20:51:57.347Z", + "createdBy": "admin", + "modifiedTimeStamp": "2023-12-20T20:51:57.347Z", + "modifiedBy": "admin", + "id": "0063116c-577c-0f44-8116-3924506c8f4a", + "links": [ + { + "method": "GET", + "rel": "self", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "type": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "alternate", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "type": "application/vnd.sas.summary" + }, + { + "method": "GET", + "rel": "up", + "href": "/catalog/instances", + "uri": "/catalog/instances", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance" + }, + { + "method": "PATCH", + "rel": "patch", + "href": "/catalog/updates/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/updates/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "type": "application/json-patch", + "responseType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "PUT", + "rel": "update", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "type": "application/vnd.sas.metadata.instance.entity", + "responseType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "DELETE", + "rel": "delete", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a" + }, + { + "method": "GET", + "rel": "definition", + "href": "/catalog/definitions/3a0d5d7b-a1c5-44c6-bfad-0d2174236172", + "uri": "/catalog/definitions/3a0d5d7b-a1c5-44c6-bfad-0d2174236172", + "type": "application/vnd.sas.metadata.definition.entity" + }, + { + "method": "GET", + "rel": "history", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a/history", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a/history", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "resource", + "href": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER?ext=sashdat", + "uri": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER?ext=sashdat", + "type": "application/json" + } + ], + "version": 1, + "instanceType": "entity", + "definitionId": "3a0d5d7b-a1c5-44c6-bfad-0d2174236172", + "name": "WATER_CLUSTER.sashdat", + "type": "casTable", + "attributes": { + "analysisTimeStamp": "2023-12-20T20:52:01.453Z", + "columnCount": 21, + "completenessPercent": 95, + "creator": "sas", + "dataLocale": "EN", + "dataSize": 10091520, + "dateCreated": "2023-12-20T08:28:18.000Z", + "dateFieldCount": 1, + "dateModified": "2023-12-20T08:28:18.500Z", + "encoding": "utf-8", + "extension": "sashdat", + "informationPrivacy": "candidate", + "isLoaded": false, + "isView": false, + "maxTimeStamp": "2015-12-31T00:00:00.000Z", + "minTimeStamp": "2014-01-01T00:00:00.000Z", + "mostImportantFields": "\"Property\",\"Lat\",\"Long\"", + "nlpTags": "DATE,INDIVIDUAL", + "nlpTerms": "st patrick,christmas,memorial day,labor day,martin luther king,new year's day,thanksgiving,veteran's day,friday", + "observationFrequency": "hourly", + "observationFrequencyField": "Date", + "reviewStatus": "none", + "rowCount": 46720, + "runId": "cebf0b96-594f-4c83-b4d8-49f8cac78e20", + "runStatus": "changed", + "sourceSystem": "CAS" + }, + "resourceId": "/dataTables/dataSources/cas~fs~cas-shared-default~fs~Samples/tables/WATER_CLUSTER?ext=sashdat" + } + ], + "relationships": [ + { + "creationTimeStamp": "2023-12-20T20:51:58.19Z", + "createdBy": "admin", + "modifiedTimeStamp": "2023-12-20T20:51:58.19Z", + "modifiedBy": "admin", + "id": "1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "links": [ + { + "method": "GET", + "rel": "self", + "href": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "uri": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "type": "application/vnd.sas.metadata.instance.relationship" + }, + { + "method": "GET", + "rel": "alternate", + "href": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "uri": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "type": "application/vnd.sas.summary" + }, + { + "method": "GET", + "rel": "up", + "href": "/catalog/instances", + "uri": "/catalog/instances", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance" + }, + { + "method": "PATCH", + "rel": "patch", + "href": "/catalog/updates/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "uri": "/catalog/updates/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "type": "application/json-patch", + "responseType": "application/vnd.sas.metadata.instance.relationship" + }, + { + "method": "PUT", + "rel": "update", + "href": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "uri": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "type": "application/vnd.sas.metadata.instance.relationship", + "responseType": "application/vnd.sas.metadata.instance.relationship" + }, + { + "method": "DELETE", + "rel": "delete", + "href": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab", + "uri": "/catalog/instances/1fdc5f84-8d4b-4976-a1bc-fd98ebd710ab" + }, + { + "method": "GET", + "rel": "definition", + "href": "/catalog/definitions/4e7ee014-5234-422b-acf1-18ca08ff1c4e", + "uri": "/catalog/definitions/4e7ee014-5234-422b-acf1-18ca08ff1c4e", + "type": "application/vnd.sas.metadata.definition.relationship" + }, + { + "method": "GET", + "rel": "endpoint1", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "type": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "endpoint2", + "href": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "uri": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465", + "type": "application/vnd.sas.metadata.instance.entity" + } + ], + "version": 1, + "instanceType": "relationship", + "definitionId": "4e7ee014-5234-422b-acf1-18ca08ff1c4e", + "type": "dataSetDataFields", + "endpoint1Id": "0063116c-577c-0f44-8116-3924506c8f4a", + "endpoint1Uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "endpoint2Id": "a278a7b0-48f2-314b-a2be-fd2753dc7465", + "endpoint2Uri": "/catalog/instances/a278a7b0-48f2-314b-a2be-fd2753dc7465" + }, + { + "creationTimeStamp": "2023-12-20T20:51:58.008Z", + "createdBy": "admin", + "modifiedTimeStamp": "2023-12-20T20:51:58.008Z", + "modifiedBy": "admin", + "id": "2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "links": [ + { + "method": "GET", + "rel": "self", + "href": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "uri": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "type": "application/vnd.sas.metadata.instance.relationship" + }, + { + "method": "GET", + "rel": "alternate", + "href": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "uri": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "type": "application/vnd.sas.summary" + }, + { + "method": "GET", + "rel": "up", + "href": "/catalog/instances", + "uri": "/catalog/instances", + "type": "application/vnd.sas.collection", + "itemType": "application/vnd.sas.metadata.instance" + }, + { + "method": "PATCH", + "rel": "patch", + "href": "/catalog/updates/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "uri": "/catalog/updates/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "type": "application/json-patch", + "responseType": "application/vnd.sas.metadata.instance.relationship" + }, + { + "method": "PUT", + "rel": "update", + "href": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "uri": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "type": "application/vnd.sas.metadata.instance.relationship", + "responseType": "application/vnd.sas.metadata.instance.relationship" + }, + { + "method": "DELETE", + "rel": "delete", + "href": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7", + "uri": "/catalog/instances/2e523ca1-3346-4a9c-9fb2-22627f895fc7" + }, + { + "method": "GET", + "rel": "definition", + "href": "/catalog/definitions/4e7ee014-5234-422b-acf1-18ca08ff1c4e", + "uri": "/catalog/definitions/4e7ee014-5234-422b-acf1-18ca08ff1c4e", + "type": "application/vnd.sas.metadata.definition.relationship" + }, + { + "method": "GET", + "rel": "endpoint1", + "href": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "type": "application/vnd.sas.metadata.instance.entity" + }, + { + "method": "GET", + "rel": "endpoint2", + "href": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "uri": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c", + "type": "application/vnd.sas.metadata.instance.entity" + } + ], + "version": 1, + "instanceType": "relationship", + "definitionId": "4e7ee014-5234-422b-acf1-18ca08ff1c4e", + "type": "dataSetDataFields", + "endpoint1Id": "0063116c-577c-0f44-8116-3924506c8f4a", + "endpoint1Uri": "/catalog/instances/0063116c-577c-0f44-8116-3924506c8f4a", + "endpoint2Id": "5569fb65-c6df-484a-897e-7a087665cd6c", + "endpoint2Uri": "/catalog/instances/5569fb65-c6df-484a-897e-7a087665cd6c" + } + ] +} \ No newline at end of file diff --git a/ingestion/tests/unit/topology/database/test_sas.py b/ingestion/tests/unit/topology/database/test_sas.py new file mode 100644 index 000000000000..70f61997dc0b --- /dev/null +++ b/ingestion/tests/unit/topology/database/test_sas.py @@ -0,0 +1,414 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test SAS using the topology +""" +import json +from pathlib import Path +from unittest import TestCase +from unittest.mock import patch + +from pydantic import AnyUrl + +from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest +from metadata.generated.schema.api.data.createDatabaseSchema import ( + CreateDatabaseSchemaRequest, +) +from metadata.generated.schema.api.data.createTable import CreateTableRequest +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceRequest, +) +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.table import Column, Table +from metadata.generated.schema.entity.services.connections.database.sasConnection import ( + SASConnection, +) +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.generated.schema.type.basic import EntityExtension, Href +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.database.sas.client import SASClient +from metadata.ingestion.source.database.sas.metadata import SasSource + +mock_sas_config = { + "source": { + "type": "sas", + "serviceName": "local_sas", + "serviceConnection": { + "config": { + "type": "SAS", + "serverHost": "http://your-server-host.org", + "username": "username", + "password": "password", + "datatables": True, + "dataTablesCustomFilter": None, + "reports": False, + "reportsCustomFilter": None, + "dataflows": False, + "dataflowsCustomFilter": None, + } + }, + "sourceConfig": {"config": {"type": "DatabaseMetadata"}}, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": { + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc" + "iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE" + "2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB" + "iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN" + "r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u" + "d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + }, + } + }, +} + +mock_search_path = ( + Path(__file__).parent.parent.parent / "resources/datasets/sas_dataset_search.json" +) +mock_view_path = ( + Path(__file__).parent.parent.parent / "resources/datasets/sas_dataset_view.json" +) +with open(mock_search_path, encoding="UTF-8") as search_file: + mock_search: dict = json.load(search_file) + +with open(mock_view_path, encoding="UTF-8") as view_file: + mock_view: dict = json.load(view_file) + + +def mock_list_assets(self, table): # pylint: disable=unused-argument + return mock_search + + +def mock_access_token( + self, base_url, user, password +): # pylint: disable=unused-argument + return "access_token" + + +def mock_get_views(self, query): # pylint: disable=unused-argument + return mock_view + + +EXPECTED_TABLE = Table( + id="0063116c-577c-0f44-8116-3924506c8f4a", + name="WATER_CLUSTER.sashdat", + displayName=None, + fullyQualifiedName="cas.cas-shared-default.Samples.WATER_CLUSTER.sashdat", + description='Last analyzed: 2023-12-20T20:52:01.453Z. Visit SAS Information Catalog for more information.', + version=0.1, + updatedAt=1703105517347, + updatedBy="admin", + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/tables/124d078d-dcf2-43a8-b59e-33bc7953f680", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/tables/0063116c-577c-0f44-8116-3924506c8f4a", + ) + ), + tableType="Regular", + columns=[ + Column( + name="Address", + displayName=None, + dataType="STRING", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="string", + description="", + fullyQualifiedName="cas.cas-shared-default.Samples.WATER_CLUSTER.sashdat.Address", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + Column( + name="Address", + displayName=None, + dataType="STRING", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="string", + description="", + fullyQualifiedName="cas.cas-shared-default.Samples.WATER_CLUSTER.sashdat.US%20Holidaty", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + ], + tableConstraints=None, + tablePartition=None, + owner=None, + databaseSchema=EntityReference( + id="4cf6ee7e-9d24-4153-9318-82aa1167259b", + type="databaseSchema", + name="Samples", + fullyQualifiedName="cas.cas-shared-default.Samples", + description="", + displayName=None, + deleted=False, + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b", + ) + ), + ), + database=EntityReference( + id="367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd", + type="database", + name="Samples", + fullyQualifiedName="cas.cas-shared-default", + description="", + displayName=None, + deleted=False, + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd", + ) + ), + ), + service=EntityReference( + id="f2ab0e7a-5224-4acb-a189-74158851733f", + type="databaseService", + name="cas", + fullyQualifiedName="cas", + description=None, + displayName=None, + deleted=False, + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f", + ) + ), + ), + serviceType="SAS", + location=None, + viewDefinition=None, + usageSummary=None, + followers=None, + joins=None, + sampleData=None, + tableProfilerConfig=None, + profile=None, + dataModel=None, + changeDescription=None, + deleted=False, + extension=EntityExtension( + __root__={ + "analysisTimeStamp": "2023-12-20T20:52:01.453Z", + "columnCount": 21, + "completenessPercent": 95, + "creator": "sas", + "dataSize": 10091520, + "dateCreated": "2023-12-20T08:28:18.000Z", + "dateModified": "2023-12-20T08:28:18.500Z", + "rowCount": 46720, + } + ), +) + + +class SASUnitTest(TestCase): + """ + Implements the necessary methods to extract + SAS Metadata Unit Test + """ + + @patch("metadata.ingestion.source.database.sas.metadata.SasSource.test_connection") + @patch.object(SASClient, "get_token", mock_access_token) + def __init__(self, method_name, test_connection) -> None: + super().__init__(method_name) + test_connection.return_value = False + self.config = OpenMetadataWorkflowConfig.parse_obj(mock_sas_config) + self.sas_source = SasSource.create( + mock_sas_config["source"], + OpenMetadata(self.config.workflowConfig.openMetadataServerConfig), + ) + self.metadata = OpenMetadata( + OpenMetadataConnection.parse_obj( + mock_sas_config["workflowConfig"]["openMetadataServerConfig"] + ) + ) + + config_ = mock_sas_config["source"]["serviceConnection"]["config"] + self.database_service = ( + mock_database_service_object + ) = self.metadata.create_or_update( + CreateDatabaseServiceRequest( + name="local_sas", + serviceType="SAS", + connection=DatabaseConnection( + config=SASConnection( + type=config_["type"], + username=config_["username"], + password=config_["password"], + serverHost=config_["serverHost"], + datatables=config_["datatables"], + dataTablesCustomFilter=config_["dataTablesCustomFilter"], + reports=config_["reports"], + reportsCustomFilter=config_["reportsCustomFilter"], + dataflows=config_["dataflows"], + dataflowsCustomFilter=config_["dataflowsCustomFilter"], + ) + ), + ) + ) + + mock_database_object = self.metadata.create_or_update( + CreateDatabaseRequest( + name="cas.cas-shared-default", + displayName=None, + description=None, + tags=None, + owner=None, + service=mock_database_service_object.fullyQualifiedName, + ) + ) + mock_database_schema_object = self.metadata.create_or_update( + CreateDatabaseSchemaRequest( + name="Samples", + database=mock_database_object.fullyQualifiedName, + ) + ) + _ = self.metadata.create_or_update( + CreateTableRequest( + name="WATER_CLUSTER.sashdat", + tableType="Regular", + columns=[ + Column( + name="Address", + displayName=None, + dataType="STRING", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="string", + description="", + fullyQualifiedName=f"{mock_database_schema_object.fullyQualifiedName}" + f'."WATER_CLUSTER.sashdat".Address', + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + Column( + name="US Holiday", + displayName=None, + dataType="STRING", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="string", + description="", + fullyQualifiedName=f"{mock_database_schema_object.fullyQualifiedName}" + f'."WATER_CLUSTER.sashdat".US%20Holiday', + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + ], + databaseSchema=mock_database_schema_object.fullyQualifiedName, + sourceUrl="None", + ) + ) + + @patch.object(SASClient, "list_assets", mock_list_assets) + @patch.object(SASClient, "get_views", mock_get_views) + def test_loaded_context(self): + """ + Testing description updated for database, databaseSchema, table + """ + _ = list(self.sas_source._iter()) + loaded_database = self.metadata.get_by_name( + entity=Database, fqn='local_sas."cas.cas-shared-default"' + ) + + assert loaded_database + assert loaded_database.name.__root__ == "cas.cas-shared-default" + + loaded_database_schema = self.metadata.get_by_name( + entity=DatabaseSchema, fqn='local_sas."cas.cas-shared-default".Samples' + ) + + assert loaded_database_schema + assert loaded_database_schema.name.__root__ == "Samples" + + loaded_table = self.metadata.get_by_name( + entity=Table, + fqn='local_sas."cas.cas-shared-default".Samples."WATER_CLUSTER.sashdat"', + fields=["extension"], + ) + + assert loaded_table + assert loaded_table.description == EXPECTED_TABLE.description + assert loaded_table.extension == EXPECTED_TABLE.extension + + # clean up added database service connector + self.metadata.delete( + entity=DatabaseService, + entity_id=self.database_service.id, + recursive=True, + hard_delete=True, + ) diff --git a/openmetadata-docs/content/v1.1.x/how-to-guides/quick-start-guide-for-admins/how-to-ingest-metadata.md b/openmetadata-docs/content/v1.1.x/how-to-guides/quick-start-guide-for-admins/how-to-ingest-metadata.md index eb959ddcf5a2..db250d4b896a 100644 --- a/openmetadata-docs/content/v1.1.x/how-to-guides/quick-start-guide-for-admins/how-to-ingest-metadata.md +++ b/openmetadata-docs/content/v1.1.x/how-to-guides/quick-start-guide-for-admins/how-to-ingest-metadata.md @@ -27,7 +27,7 @@ href="/connectors"%} Refer to the Docs to ingest metadata from multiple sources - Databases, Dashboards, Pipelines, ML Models, Messaging, Storage, as well as Metadata services. {%/inlineCallout%} -- **Database Services:** [Athena](/connectors/database/athena), [AzureSQL](/connectors/database/azuresql), [BigQuery](/connectors/database/bigquery), [Clickhouse](/connectors/database/clickhouse), [Databricks](/connectors/database/databricks), [Datalake](/connectors/database/datalake), [DB2](/connectors/database/db2), [DeltaLake](/connectors/database/deltalake), [Domo Database](/connectors/database/domo-database), [Druid](/connectors/database/druid), [DynamoDB](/connectors/database/dynamodb), [Glue](/connectors/database/glue), [Hive](/connectors/database/hive), [Impala](/connectors/database/impala), [MariaDB](/connectors/database/mariadb), [MongoDB](/connectors/database/mongodb), [MSSQL](/connectors/database/mssql), [MySQL](/connectors/database/mysql), [Oracle](/connectors/database/oracle), [PinotDB](/connectors/database/pinotdb), [Postgres](/connectors/database/postgres), [Presto](/connectors/database/presto), [Redshift](/connectors/database/redshift), [Salesforce](/connectors/database/salesforce), [SAP Hana](/connectors/database/sap-hana), [SingleStore](/connectors/database/singlestore), [Snowflake](/connectors/database/snowflake), [SQLite](/connectors/database/sqlite), [Trino](/connectors/database/trino), [Vertica](/connectors/database/vertica), and [SAS](/connectors/database/sas). +- **Database Services:** [Athena](/connectors/database/athena), [AzureSQL](/connectors/database/azuresql), [BigQuery](/connectors/database/bigquery), [Clickhouse](/connectors/database/clickhouse), [Databricks](/connectors/database/databricks), [Datalake](/connectors/database/datalake), [DB2](/connectors/database/db2), [DeltaLake](/connectors/database/deltalake), [Domo Database](/connectors/database/domo-database), [Druid](/connectors/database/druid), [DynamoDB](/connectors/database/dynamodb), [Glue](/connectors/database/glue), [Hive](/connectors/database/hive), [Impala](/connectors/database/impala), [MariaDB](/connectors/database/mariadb), [MongoDB](/connectors/database/mongodb), [MSSQL](/connectors/database/mssql), [MySQL](/connectors/database/mysql), [Oracle](/connectors/database/oracle), [PinotDB](/connectors/database/pinotdb), [Postgres](/connectors/database/postgres), [Presto](/connectors/database/presto), [Redshift](/connectors/database/redshift), [Salesforce](/connectors/database/salesforce), [SAP Hana](/connectors/database/sap-hana), [SingleStore](/connectors/database/singlestore), [Snowflake](/connectors/database/snowflake), [SQLite](/connectors/database/sqlite), [Trino](/connectors/database/trino), and [Vertica](/connectors/database/vertica). - **Dashboard Services:** [Domo Dashboard](/connectors/dashboard/domo-dashboard), [Looker](/connectors/dashboard/looker), [Metabase](/connectors/dashboard/metabase), [Mode](/connectors/dashboard/mode), [PowerBI](/connectors/dashboard/powerbi), [Qlik Sense](/connectors/dashboard/qliksense), [QuickSight](/connectors/dashboard/quicksight), [Redash](/connectors/dashboard/redash), [Superset](/connectors/dashboard/superset), and [Tableau](/connectors/dashboard/tableau). diff --git a/openmetadata-docs/content/v1.2.x/connectors/database/index.md b/openmetadata-docs/content/v1.2.x/connectors/database/index.md index 4bc3372d0af5..7e66479003d3 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/database/index.md +++ b/openmetadata-docs/content/v1.2.x/connectors/database/index.md @@ -35,7 +35,6 @@ This is the supported list of connectors for Database Services: - [Trino](/connectors/database/trino) - [Unity Catalog](/connectors/database/unity-catalog) - [Vertica](/connectors/database/vertica) -- [SAS](/connectors/database/sas) If you have a request for a new connector, don't hesitate to reach out in [Slack](https://slack.open-metadata.org/) or open a [feature request](https://github.com/open-metadata/OpenMetadata/issues/new/choose) in our GitHub repo. diff --git a/openmetadata-docs/content/v1.2.x/connectors/index.md b/openmetadata-docs/content/v1.2.x/connectors/index.md index d5e59059d3a6..62271cf12ade 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/index.md +++ b/openmetadata-docs/content/v1.2.x/connectors/index.md @@ -64,7 +64,6 @@ the following docs to run the Ingestion Framework in any orchestrator externally - [Trino](/connectors/database/trino) - [Unity Catalog](/connectors/database/unity-catalog) - [Vertica](/connectors/database/vertica) -- [SAS](/connectors/database/sas) ## Dashboard Services diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/index.md index 89b221946d55..c87c0c76f2d7 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/index.md @@ -29,6 +29,7 @@ This is the supported list of connectors for Database Services: - [Presto](/connectors/database/presto) - [Redshift](/connectors/database/redshift) - [Salesforce](/connectors/database/salesforce) +- [SAS](/connectors/database/sas) - [SingleStore](/connectors/database/singlestore) - [Snowflake](/connectors/database/snowflake) - [Trino](/connectors/database/trino) diff --git a/openmetadata-docs/content/v1.2.x/connectors/database/sas/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/sas/index.md similarity index 93% rename from openmetadata-docs/content/v1.2.x/connectors/database/sas/index.md rename to openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/sas/index.md index 39c1ec29ed7e..30c0f7367b00 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/database/sas/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/sas/index.md @@ -31,7 +31,7 @@ Prepare the SAS Service and configure the Ingestion: - **ServerHost**: Host and port of the SAS Viya deployment. - **Username**: Username to connect to SAS Viya. This user should have privileges to read all the metadata in SAS Information Catalog. - **Password**: Password to connect to SAS Viya. -- **Filter**: A filter expression specifying items for import. For more information see https://developer.sas.com/apis/rest/DataManagement/#catalog +- **Filter**: A filter expression specifying items for import. For more information [see](https://developer.sas.com/apis/rest/DataManagement/#catalog) {% /extraContent %} diff --git a/openmetadata-docs/content/v1.2.x/connectors/database/sas/yaml.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/sas/yaml.md similarity index 87% rename from openmetadata-docs/content/v1.2.x/connectors/database/sas/yaml.md rename to openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/sas/yaml.md index e6e798bce830..730c719de62c 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/database/sas/yaml.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/sas/yaml.md @@ -69,6 +69,13 @@ The workflow is modeled around the following {% /codeInfo %} +{% codeInfo srNumber=15 %} + +**filter**: A filter expression specifying items for import. For more information [see](https://developer.sas.com/apis/rest/DataManagement/#catalog-search) + +{% /codeInfo %} + + #### Sink Configuration {% codeInfo srNumber=18 %} @@ -99,6 +106,15 @@ source: ```yaml {% srNumber=14 %} password: password ``` +```yaml {% srNumber=15 %} + datatables: True + dataTablesCustomFilter: None + reports: False + reportsCustomFilter: None + dataflows: False + dataflowsCustomFilter: None +``` +```yaml sourceConfig: config: type: DatabaseMetadata diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/index.md index 62271cf12ade..7cf5dfd7e7a6 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/index.md @@ -58,6 +58,7 @@ the following docs to run the Ingestion Framework in any orchestrator externally - [Redshift](/connectors/database/redshift) - [Salesforce](/connectors/database/salesforce) - [SAP Hana](/connectors/database/sap-hana) +- [SAS](/connectors/database/sas) - [SingleStore](/connectors/database/singlestore) - [Snowflake](/connectors/database/snowflake) - [SQLite](/connectors/database/sqlite) diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/how-to-guides/admin-guide/how-to-ingest-metadata.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/how-to-guides/admin-guide/how-to-ingest-metadata.md index 30c6abbf7a59..b361646000e2 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/how-to-guides/admin-guide/how-to-ingest-metadata.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/how-to-guides/admin-guide/how-to-ingest-metadata.md @@ -27,7 +27,7 @@ href="/connectors"%} Refer to the Docs to ingest metadata from multiple sources - Databases, Dashboards, Pipelines, ML Models, Messaging, Storage, as well as Metadata services. {%/inlineCallout%} -- **Database Services:** [Athena](/connectors/database/athena), [AzureSQL](/connectors/database/azuresql), [BigQuery](/connectors/database/bigquery), [Clickhouse](/connectors/database/clickhouse), [Databricks](/connectors/database/databricks), [Datalake](/connectors/database/datalake), [DB2](/connectors/database/db2), [DeltaLake](/connectors/database/deltalake), [Domo Database](/connectors/database/domo-database), [Druid](/connectors/database/druid), [DynamoDB](/connectors/database/dynamodb), [Glue](/connectors/database/glue), [Hive](/connectors/database/hive), [Impala](/connectors/database/impala), [MariaDB](/connectors/database/mariadb), [MongoDB](/connectors/database/mongodb), [MSSQL](/connectors/database/mssql), [MySQL](/connectors/database/mysql), [Oracle](/connectors/database/oracle), [PinotDB](/connectors/database/pinotdb), [Postgres](/connectors/database/postgres), [Presto](/connectors/database/presto), [Redshift](/connectors/database/redshift), [Salesforce](/connectors/database/salesforce), [SAP Hana](/connectors/database/sap-hana), [SingleStore](/connectors/database/singlestore), [Snowflake](/connectors/database/snowflake), [SQLite](/connectors/database/sqlite), [Trino](/connectors/database/trino), and [Vertica](/connectors/database/vertica). +- **Database Services:** [Athena](/connectors/database/athena), [AzureSQL](/connectors/database/azuresql), [BigQuery](/connectors/database/bigquery), [Clickhouse](/connectors/database/clickhouse), [Databricks](/connectors/database/databricks), [Datalake](/connectors/database/datalake), [DB2](/connectors/database/db2), [DeltaLake](/connectors/database/deltalake), [Domo Database](/connectors/database/domo-database), [Druid](/connectors/database/druid), [DynamoDB](/connectors/database/dynamodb), [Glue](/connectors/database/glue), [Hive](/connectors/database/hive), [Impala](/connectors/database/impala), [MariaDB](/connectors/database/mariadb), [MongoDB](/connectors/database/mongodb), [MSSQL](/connectors/database/mssql), [MySQL](/connectors/database/mysql), [Oracle](/connectors/database/oracle), [PinotDB](/connectors/database/pinotdb), [Postgres](/connectors/database/postgres), [Presto](/connectors/database/presto), [Redshift](/connectors/database/redshift), [Salesforce](/connectors/database/salesforce), [SAP Hana](/connectors/database/sap-hana), [SAS](/connectors/database/sas), [SingleStore](/connectors/database/singlestore), [Snowflake](/connectors/database/snowflake), [SQLite](/connectors/database/sqlite), [Trino](/connectors/database/trino), and [Vertica](/connectors/database/vertica). - **Dashboard Services:** [Domo Dashboard](/connectors/dashboard/domo-dashboard), [Looker](/connectors/dashboard/looker), [Metabase](/connectors/dashboard/metabase), [Mode](/connectors/dashboard/mode), [PowerBI](/connectors/dashboard/powerbi), [Qlik Sense](/connectors/dashboard/qliksense), [QuickSight](/connectors/dashboard/quicksight), [Redash](/connectors/dashboard/redash), [Superset](/connectors/dashboard/superset), and [Tableau](/connectors/dashboard/tableau). diff --git a/openmetadata-docs/content/v1.2.x/main-concepts/metadata-standard/schemas/entity/services/connections/database/sasConnection.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/main-concepts/metadata-standard/schemas/entity/services/connections/database/sasConnection.md similarity index 78% rename from openmetadata-docs/content/v1.2.x/main-concepts/metadata-standard/schemas/entity/services/connections/database/sasConnection.md rename to openmetadata-docs/content/v1.3.x-SNAPSHOT/main-concepts/metadata-standard/schemas/entity/services/connections/database/sasConnection.md index 1bc67233a1d9..66235c0e2c70 100644 --- a/openmetadata-docs/content/v1.2.x/main-concepts/metadata-standard/schemas/entity/services/connections/database/sasConnection.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/main-concepts/metadata-standard/schemas/entity/services/connections/database/sasConnection.md @@ -13,6 +13,8 @@ slug: /main-concepts/metadata-standard/schemas/entity/services/connections/datab - **`username`** *(string)*: Username to connect to SAS Viya. - **`password`** *(string)*: Password to connect to SAS Viya. - **`serverHost`** *(string)*: Hostname of SAS Viya deployment. +- **`filter`** *(string)*: A filter expression specifying items for import. For more information [see](https://developer.sas.com/apis/rest/DataManagement/#catalog-search) + ## Definitions - **`sasType`** *(string)*: Service type. Must be one of: `['SAS']`. Default: `SAS`. diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md index 0628990af310..ebdf0a9ba2b4 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md @@ -498,6 +498,10 @@ site_menu: url: /connectors/database/sap-hana - category: Connectors / Database / SAP Hana / Run Externally url: /connectors/database/sap-hana/yaml + - category: Connectors / Database / SAS + url: /connectors/database/sas + - category: Connectors / Database / SAS / Run Externally + url: /connectors/database/sas/yaml - category: Connectors / Database / SingleStore url: /connectors/database/singlestore - category: Connectors / Database / SingleStore / Run Externally diff --git a/openmetadata-docs/images/v1.2/connectors/sas/add-new-service.png b/openmetadata-docs/images/v1.2/connectors/sas/add-new-service.png deleted file mode 100644 index c5246e7e6d35..000000000000 Binary files a/openmetadata-docs/images/v1.2/connectors/sas/add-new-service.png and /dev/null differ diff --git a/openmetadata-docs/images/v1.2/connectors/sas/select-service.png b/openmetadata-docs/images/v1.2/connectors/sas/select-service.png deleted file mode 100644 index a4cffe85fe48..000000000000 Binary files a/openmetadata-docs/images/v1.2/connectors/sas/select-service.png and /dev/null differ diff --git a/openmetadata-docs/images/v1.2/connectors/sas/service-connection.png b/openmetadata-docs/images/v1.2/connectors/sas/service-connection.png deleted file mode 100644 index 9dd298191472..000000000000 Binary files a/openmetadata-docs/images/v1.2/connectors/sas/service-connection.png and /dev/null differ diff --git a/openmetadata-docs/images/v1.3/connectors/sas/add-new-service.png b/openmetadata-docs/images/v1.3/connectors/sas/add-new-service.png new file mode 100644 index 000000000000..b5953248602d Binary files /dev/null and b/openmetadata-docs/images/v1.3/connectors/sas/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.3/connectors/sas/select-service.png b/openmetadata-docs/images/v1.3/connectors/sas/select-service.png new file mode 100644 index 000000000000..7681aa060835 Binary files /dev/null and b/openmetadata-docs/images/v1.3/connectors/sas/select-service.png differ diff --git a/openmetadata-docs/images/v1.3/connectors/sas/service-connection.png b/openmetadata-docs/images/v1.3/connectors/sas/service-connection.png new file mode 100644 index 000000000000..cd4f035f8760 Binary files /dev/null and b/openmetadata-docs/images/v1.3/connectors/sas/service-connection.png differ