Minor: Fixed dbtcloud test connection and improved docs (#18408)

ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/client.py

@@ -63,6 +63,30 @@ def __init__(self, config: DBTCloudConnection):
         self.client = REST(client_config)
         self.graphql_client = REST(graphql_client_config)
 
+    def test_get_jobs(self) -> Optional[List[DBTJob]]:
+        """
+        test fetch jobs for an account in dbt cloud
+        """
+        job_path = f"{self.config.jobId}/" if self.config.jobId else ""
+        result = self.client.get(f"/accounts/{self.config.accountId}/jobs/{job_path}")
+        job_list = (
+            [DBTJob.model_validate(result["data"])]
+            if self.config.jobId
+            else DBTJobList.model_validate(result).Jobs
+        )
+        return job_list
+
+    def test_get_runs(self, job_id: int) -> Optional[List[DBTRun]]:
+        """
+        test fetch runs for a job in dbt cloud
+        """
+        result = self.client.get(
+            f"/accounts/{self.config.accountId}/runs/",
+            data={"job_definition_id": job_id},
+        )
+        run_list = DBTRunList.model_validate(result).Runs
+        return run_list
+
     def get_jobs(self) -> Optional[List[DBTJob]]:
         """
         list jobs for an account in dbt cloud
@@ -73,7 +97,11 @@ def get_jobs(self) -> Optional[List[DBTJob]]:
                 f"/accounts/{self.config.accountId}/jobs/{job_path}"
             )
             if result:
-                job_list = DBTJobList(**result).Jobs
+                job_list = (
+                    [DBTJob.model_validate(result.get("data"))]
+                    if self.config.jobId
+                    else DBTJobList.model_validate(result).Jobs
+                )
                 return job_list
         except Exception as exc:
             logger.debug(traceback.format_exc())
@@ -91,7 +119,7 @@ def get_runs(self, job_id: int) -> Optional[List[DBTRun]]:
                 data={"job_definition_id": job_id},
             )
             if result:
-                run_list = DBTRunList(**result).Runs
+                run_list = DBTRunList.model_validate(result).Runs
                 return run_list
         except Exception as exc:
             logger.debug(traceback.format_exc())
@@ -112,7 +140,7 @@ def get_model_details(self, job_id: int, run_id: int):
             result = self.graphql_client.post("", json=query_params)
 
             if result.get("data") and result["data"].get("job"):
-                model_list = DBTModelList(**result["data"]["job"]).models
+                model_list = DBTModelList.model_validate(result["data"]["job"]).models
                 return model_list
 
         except Exception as exc:
@@ -133,7 +161,7 @@ def get_models_and_seeds_details(self, job_id: int, run_id: int):
             result = self.graphql_client.post("", json=query_params)
 
             if result.get("data") and result["data"].get("job"):
-                result = DBTModelList(**result["data"]["job"])
+                result = DBTModelList.model_validate(result["data"]["job"])
                 parents_list = result.models + result.seeds
                 return parents_list
 

ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py

@@ -53,8 +53,8 @@ def test_connection(
     job_id = int(service_connection.jobId) if service_connection.jobId else 0
 
     test_fn = {
-        "GetJobs": client.get_jobs,
-        "GetRuns": partial(client.get_runs, job_id=job_id),
+        "GetJobs": client.test_get_jobs,
+        "GetRuns": partial(client.test_get_runs, job_id=job_id),
     }
 
     return test_connection_steps(

openmetadata-docs/content/v1.5.x/connectors/pipeline/dbtcloud/index.md

@@ -63,11 +63,11 @@ To know more about permissions required refer [here](https://docs.getdbt.com/doc
 
 #### Connection Details
 
-- **Host**: DBT cloud Access URL eg.`https://abc12.us1.dbt.com`. Go to your dbt cloud account settings to know your Access URL.
+- **Host**: DBT cloud Access URL eg.`https://abc12.us1.dbt.com`. Go to your dbt cloud account settings then go to the Access URLs section. In there you will find various URLs we need the `Access URL` from that section as the Host. For more info visit [here](https://docs.getdbt.com/docs/cloud/about-cloud/access-regions-ip-addresses#api-access-urls).
 
-- **Discovery API URL** : DBT cloud Access URL eg. `https://metadata.cloud.getdbt.com/graphql`. Go to your dbt cloud account settings to know your Discovery API url. Make sure you have `/graphql` at the end of your URL.
+- **Discovery API URL** : DBT cloud Discovery API URL eg. `https://abc12.metadata.us1.dbt.com/graphql`. Go to your dbt cloud account settings where you found your Access URL. In there scroll down to find `Discovery API URL` . If your `Discovery API URL` doesn't contain the `/graphql` at the end please add it. Make sure you have `/graphql` at the end of your URL. Note that `Semantic Layer GraphQL API URL` is different from `Discovery API URL`.
 
-- **Account Id** : The Account ID of your DBT cloud Project. Go to your dbt cloud account settings to know your Account Id. This will be a numeric value but in openmetadata we parse it as a string.
+- **Account Id** : The Account ID of your DBT cloud Project. Go to your dbt cloud account settings then in the `Account information` you will find `Account ID`. This will be a numeric value but in openmetadata we parse it as a string.
 
 - **Job Id** : Optional. The Job ID of your DBT cloud Job in your Project to fetch metadata for. Look for the segment after "jobs" in the URL. For instance, in a URL like `https://cloud.getdbt.com/accounts/123/projects/87477/jobs/73659994`, the job ID is `73659994`. This will be a numeric value but in openmetadata we parse it as a string. If not passed all Jobs under the Account id will be ingested.
 

openmetadata-docs/content/v1.5.x/connectors/pipeline/dbtcloud/yaml.md

@@ -52,19 +52,19 @@ This is a sample config for dbt Cloud:
 
 {% codeInfo srNumber=1 %}
 
-**host**: DBT cloud Access URL eg.`https://abc12.us1.dbt.com`. Go to your dbt cloud account settings to know your Access URL.
+**host**: DBT cloud Access URL eg.`https://abc12.us1.dbt.com`. Go to your dbt cloud account settings then go to the Access URLs section. In there you will find various URLs we need the `Access URL` from that section as the Host. For more info visit [here](https://docs.getdbt.com/docs/cloud/about-cloud/access-regions-ip-addresses#api-access-urls).
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=2 %}
 
-**discoveryAPI**: DBT cloud Access URL eg. `https://metadata.cloud.getdbt.com/graphql`. Go to your dbt cloud account settings to know your Discovery API url.
+**discoveryAPI**: DBT cloud Discovery API URL eg. `https://abc12.metadata.us1.dbt.com/graphql`. Go to your dbt cloud account settings where you found your Access URL. In there scroll down to find `Discovery API URL` . If your `Discovery API URL` doesn't contain the `/graphql` at the end please add it. Make sure you have `/graphql` at the end of your URL. Note that `Semantic Layer GraphQL API URL` is different from `Discovery API URL`.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=3 %}
 
-**accountId**: The Account ID of your DBT cloud Project. Go to your dbt cloud account settings to know your Account Id. This will be a numeric value but in openmetadata we parse it as a string.
+**accountId**: The Account ID of your DBT cloud Project. Go to your dbt cloud account settings then in the `Account information` you will find `Account ID`. This will be a numeric value but in openmetadata we parse it as a string.
 
 {% /codeInfo %}
 

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/DBTCloud.md

@@ -12,17 +12,17 @@ You can find further information on the dbtcloud connector in the [docs](https:/
 ## Connection Details
 $$section
 ### Host $(id="host")
-DBT cloud Access URL eg.`https://abc12.us1.dbt.com`. Go to your dbt cloud account settings to know your Access URL.
+DBT cloud Access URL eg.`https://abc12.us1.dbt.com`. Go to your dbt cloud account settings then go to the Access URLs section. In there you will find various URLs we need the `Access URL` from that section as the Host. For more info visit [here](https://docs.getdbt.com/docs/cloud/about-cloud/access-regions-ip-addresses#api-access-urls).
 $$
 
 $$section
 ### Discovery API URL $(id="discoveryAPI")
-DBT cloud Access URL eg. `https://metadata.cloud.getdbt.com/graphql`. Go to your dbt cloud account settings to know your Discovery API url.
+DBT cloud Discovery API URL eg. `https://abc12.metadata.us1.dbt.com/graphql`. Go to your dbt cloud account settings where you found your Access URL. In there scroll down to find `Discovery API URL` . If your `Discovery API URL` doesn't contain the `/graphql` at the end please add it. Make sure you have `/graphql` at the end of your URL. Note that `Semantic Layer GraphQL API URL` is different from `Discovery API URL`.
 $$
 
 $$section
 ### Account Id $(id="accountId")
-The Account ID of your DBT cloud Project. Go to your dbt cloud account settings to know your Account Id. This will be a numeric value but in openmetadata we parse it as a string.
+The Account ID of your DBT cloud Project. Go to your dbt cloud account settings then in the `Account information` you will find `Account ID`. This will be a numeric value but in openmetadata we parse it as a string.
 $$
 
 $$section