diff --git a/catalog/resource_catalog.go b/catalog/resource_catalog.go index 9a89352a57..7353c0bafa 100644 --- a/catalog/resource_catalog.go +++ b/catalog/resource_catalog.go @@ -27,15 +27,16 @@ func ucDirectoryPathSlashAndEmptySuppressDiff(k, old, new string, d *schema.Reso } type CatalogInfo struct { - Name string `json:"name"` - Comment string `json:"comment,omitempty"` - StorageRoot string `json:"storage_root,omitempty" tf:"force_new"` - ProviderName string `json:"provider_name,omitempty" tf:"force_new,conflicts:storage_root"` - ShareName string `json:"share_name,omitempty" tf:"force_new,conflicts:storage_root"` - Properties map[string]string `json:"properties,omitempty"` - Owner string `json:"owner,omitempty" tf:"computed"` - IsolationMode string `json:"isolation_mode,omitempty" tf:"computed"` - MetastoreID string `json:"metastore_id,omitempty" tf:"computed"` + Name string `json:"name"` + Comment string `json:"comment,omitempty"` + StorageRoot string `json:"storage_root,omitempty" tf:"force_new"` + ProviderName string `json:"provider_name,omitempty" tf:"force_new,conflicts:storage_root"` + ShareName string `json:"share_name,omitempty" tf:"force_new,conflicts:storage_root"` + ConnectionName string `json:"connection_name,omitempty" tf:"force_new,conflicts:storage_root"` + Properties map[string]string `json:"properties,omitempty"` + Owner string `json:"owner,omitempty" tf:"computed"` + IsolationMode string `json:"isolation_mode,omitempty" tf:"computed"` + MetastoreID string `json:"metastore_id,omitempty" tf:"computed"` } func ResourceCatalog() *schema.Resource { diff --git a/catalog/resource_connection.go b/catalog/resource_connection.go new file mode 100644 index 0000000000..1e6a8a1f10 --- /dev/null +++ b/catalog/resource_connection.go @@ -0,0 +1,121 @@ +package catalog + +import ( + "context" + + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/terraform-provider-databricks/common" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "golang.org/x/exp/slices" +) + +// This structure contains the fields of catalog.UpdateConnection and catalog.CreateConnection +// We need to create this because we need Owner, FullNameArg, SchemaName and CatalogName which aren't present in a single of them. +// We also need to annotate tf:"computed" for the Owner field. +type ConnectionInfo struct { + // User-provided free-form text description. + Comment string `json:"comment,omitempty" tf:"force_new"` + // The type of connection. + ConnectionType string `json:"connection_type" tf:"force_new"` + // Unique identifier of parent metastore. + MetastoreId string `json:"metastore_id,omitempty" tf:"computed"` + // Name of the connection. + Name string `json:"name"` + // Name of the connection. + NameArg string `json:"-" url:"-"` + // A map of key-value properties attached to the securable. + Options map[string]string `json:"options" tf:"sensitive"` + // Username of current owner of the connection. + Owner string `json:"owner,omitempty" tf:"force_new,suppress_diff"` + // An object containing map of key-value properties attached to the + // connection. + Properties map[string]string `json:"properties,omitempty" tf:"force_new"` + // If the connection is read only. + ReadOnly bool `json:"read_only,omitempty" tf:"force_new,computed"` +} + +var sensitiveOptions = []string{"user", "password", "personalAccessToken", "access_token", "client_secret", "OAuthPvtKey"} + +func ResourceConnection() *schema.Resource { + s := common.StructToSchema(ConnectionInfo{}, + func(m map[string]*schema.Schema) map[string]*schema.Schema { + return m + }) + pi := common.NewPairID("metastore_id", "name").Schema( + func(m map[string]*schema.Schema) map[string]*schema.Schema { + return s + }) + return common.Resource{ + Schema: s, + Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + var createConnectionRequest catalog.CreateConnection + common.DataToStructPointer(d, s, &createConnectionRequest) + conn, err := w.Connections.Create(ctx, createConnectionRequest) + if err != nil { + return err + } + d.Set("metastore_id", conn.MetastoreId) + pi.Pack(d) + return nil + }, + Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + _, connName, err := pi.Unpack(d) + if err != nil { + return err + } + conn, err := w.Connections.GetByNameArg(ctx, connName) + if err != nil { + return err + } + // We need to preserve original sensitive options as API doesn't return them + var cOrig catalog.CreateConnection + common.DataToStructPointer(d, s, &cOrig) + for key, element := range cOrig.Options { + if slices.Contains(sensitiveOptions, key) { + conn.Options[key] = element + } + } + return common.StructToData(conn, s, d) + }, + Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + var updateConnectionRequest catalog.UpdateConnection + common.DataToStructPointer(d, s, &updateConnectionRequest) + _, connName, err := pi.Unpack(d) + updateConnectionRequest.NameArg = connName + if err != nil { + return err + } + conn, err := w.Connections.Update(ctx, updateConnectionRequest) + if err != nil { + return err + } + // We need to repack the Id as the name may have changed + d.Set("name", conn.Name) + pi.Pack(d) + return nil + }, + Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + _, connName, err := pi.Unpack(d) + if err != nil { + return err + } + return w.Connections.DeleteByNameArg(ctx, connName) + }, + }.ToResource() +} diff --git a/catalog/resource_connection_test.go b/catalog/resource_connection_test.go new file mode 100644 index 0000000000..9ade5fdd7f --- /dev/null +++ b/catalog/resource_connection_test.go @@ -0,0 +1,327 @@ +package catalog + +import ( + "net/http" + "testing" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/terraform-provider-databricks/qa" + "github.com/stretchr/testify/assert" +) + +func TestConnectionsCornerCases(t *testing.T) { + qa.ResourceCornerCases(t, ResourceExternalLocation()) +} + +func TestConnectionsCreate(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: http.MethodPost, + Resource: "/api/2.1/unity-catalog/connections", + ExpectedRequest: catalog.CreateConnection{ + Name: "testConnectionName", + ConnectionType: catalog.ConnectionType("testConnectionType"), + Comment: "This is a test comment.", + Options: map[string]string{ + "host": "test.com", + }, + Properties: map[string]string{ + "purpose": "testing", + }, + Owner: "InitialOwner", + }, + Response: catalog.ConnectionInfo{ + Name: "testConnectionName", + ConnectionType: catalog.ConnectionType("testConnectionType"), + Comment: "This is a test comment.", + FullName: "testConnectionName", + MetastoreId: "abc", + Owner: "InitialOwner", + Options: map[string]string{ + "host": "test.com", + }, + Properties: map[string]string{ + "purpose": "testing", + }, + }, + }, + { + Method: http.MethodGet, + Resource: "/api/2.1/unity-catalog/connections/testConnectionName?", + Response: catalog.ConnectionInfo{ + Name: "testConnectionName", + ConnectionType: catalog.ConnectionType("testConnectionType"), + Comment: "This is a test comment.", + FullName: "testConnectionName", + Owner: "InitialOwner", + MetastoreId: "abc", + Options: map[string]string{ + "host": "test.com", + }, + Properties: map[string]string{ + "purpose": "testing", + }, + }, + }, + }, + Resource: ResourceConnection(), + Create: true, + HCL: ` + name = "testConnectionName" + connection_type = "testConnectionType" + options = { + host = "test.com" + } + properties = { + purpose = "testing" + } + comment = "This is a test comment." + owner = "InitialOwner" + `, + }.Apply(t) + assert.NoError(t, err) + assert.Equal(t, "testConnectionName", d.Get("name")) + assert.Equal(t, "testConnectionType", d.Get("connection_type")) + assert.Equal(t, "This is a test comment.", d.Get("comment")) + assert.Equal(t, map[string]interface{}{"host": "test.com"}, d.Get("options")) + assert.Equal(t, map[string]interface{}{"purpose": "testing"}, d.Get("properties")) +} + +func TestConnectionsCreate_Error(t *testing.T) { + _, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: http.MethodPost, + Resource: "/api/2.1/unity-catalog/connections", + ExpectedRequest: catalog.CreateConnection{ + Name: "testConnectionName", + ConnectionType: catalog.ConnectionType("testConnectionType"), + Comment: "This is a test comment.", + Options: map[string]string{ + "host": "test.com", + }, + Owner: "testOwner", + }, + Response: apierr.APIErrorBody{ + ErrorCode: "SERVER_ERROR", + Message: "Something unexpected happened", + }, + Status: 500, + }, + }, + Resource: ResourceConnection(), + Create: true, + HCL: ` + name = "testConnectionName" + owner = "testOwner" + connection_type = "testConnectionType" + options = { + host = "test.com" + } + comment = "This is a test comment." + `, + }.Apply(t) + qa.AssertErrorStartsWith(t, err, "Something unexpected") +} + +func TestConnectionsRead(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: http.MethodGet, + Resource: "/api/2.1/unity-catalog/connections/testConnectionName?", + Response: catalog.ConnectionInfo{ + Name: "testConnectionName", + ConnectionType: catalog.ConnectionType("testConnectionType"), + Comment: "This is a test comment.", + FullName: "testConnectionName", + MetastoreId: "abc", + Options: map[string]string{ + "host": "test.com", + }, + }, + }, + }, + Resource: ResourceConnection(), + Read: true, + ID: "abc|testConnectionName", + HCL: ` + name = "testConnectionName" + connection_type = "testConnectionType" + options = { + host = "test.com" + } + comment = "This is a test comment." + `, + }.Apply(t) + assert.NoError(t, err) + assert.Equal(t, "testConnectionName", d.Get("name")) + assert.Equal(t, "testConnectionType", d.Get("connection_type")) + assert.Equal(t, "This is a test comment.", d.Get("comment")) + assert.Equal(t, map[string]interface{}{"host": "test.com"}, d.Get("options")) +} + +func TestConnectionRead_Error(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.1/unity-catalog/connections/testConnectionName?", + Response: apierr.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, + }, + }, + Resource: ResourceConnection(), + Read: true, + ID: "abc|testConnectionName", + }.Apply(t) + qa.AssertErrorStartsWith(t, err, "Internal error happened") + assert.Equal(t, "abc|testConnectionName", d.Id(), "Id should not be empty for error reads") +} + +func TestConnectionsUpdate(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: http.MethodGet, + Resource: "/api/2.1/unity-catalog/connections/testConnectionName?", + Response: catalog.ConnectionInfo{ + Name: "testConnectionName", + ConnectionType: catalog.ConnectionType("testConnectionType"), + MetastoreId: "abc", + Comment: "testComment", + }, + }, + { + Method: http.MethodPatch, + Resource: "/api/2.1/unity-catalog/connections/testConnectionName", + ExpectedRequest: catalog.UpdateConnection{ + Name: "testConnectionNameNew", + Options: map[string]string{ + "host": "test.com", + }, + }, + Response: catalog.ConnectionInfo{ + Name: "testConnectionNameNew", + ConnectionType: catalog.ConnectionType("testConnectionType"), + Comment: "testComment", + MetastoreId: "abc", + Options: map[string]string{ + "host": "test.com", + }, + }, + }, + { + Method: http.MethodGet, + Resource: "/api/2.1/unity-catalog/connections/testConnectionNameNew?", + Response: catalog.ConnectionInfo{ + Name: "testConnectionNameNew", + ConnectionType: catalog.ConnectionType("testConnectionType"), + Comment: "testComment", + MetastoreId: "abc", + Options: map[string]string{ + "host": "test.com", + }, + }, + }, + }, + Resource: ResourceConnection(), + Update: true, + ID: "abc|testConnectionName", + InstanceState: map[string]string{ + "connection_type": "testConnectionType", + "comment": "testComment", + }, + HCL: ` + name = "testConnectionNameNew" + connection_type = "testConnectionType" + comment = "testComment" + options = { + host = "test.com" + } + `, + }.Apply(t) + assert.NoError(t, err) + assert.Equal(t, "testConnectionNameNew", d.Get("name")) + assert.Equal(t, "testConnectionType", d.Get("connection_type")) + assert.Equal(t, "testComment", d.Get("comment")) +} + +func TestConnectionUpdate_Error(t *testing.T) { + _, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: http.MethodPatch, + Resource: "/api/2.1/unity-catalog/connections/testConnectionName", + ExpectedRequest: catalog.UpdateConnection{ + Name: "testConnectionNameNew", + Options: map[string]string{ + "host": "test.com", + }, + }, + Response: apierr.APIErrorBody{ + ErrorCode: "SERVER_ERROR", + Message: "Something unexpected happened", + }, + Status: 500, + }, + }, + Resource: ResourceConnection(), + Update: true, + ID: "abc|testConnectionName", + InstanceState: map[string]string{ + "connection_type": "testConnectionType", + "comment": "testComment", + }, + HCL: ` + name = "testConnectionNameNew" + connection_type = "testConnectionType" + options = { + host = "test.com" + } + comment = "testComment" + `, + }.Apply(t) + qa.AssertErrorStartsWith(t, err, "Something unexpected") +} + +func TestConnectionDelete(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: http.MethodDelete, + Resource: "/api/2.1/unity-catalog/connections/testConnectionName?", + }, + }, + Resource: ResourceConnection(), + Delete: true, + ID: "abc|testConnectionName", + }.Apply(t) + assert.NoError(t, err) + assert.Equal(t, "abc|testConnectionName", d.Id()) +} + +func TestConnectionDelete_Error(t *testing.T) { + qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: http.MethodDelete, + Resource: "/api/2.1/unity-catalog/connections/testConnectionName?", + Response: apierr.APIErrorBody{ + ErrorCode: "INVALID_STATE", + Message: "Something went wrong", + }, + Status: 400, + }, + }, + Resource: ResourceConnection(), + Delete: true, + Removed: true, + ID: "abc|testConnectionName", + }.ExpectError(t, "Something went wrong") +} diff --git a/docs/resources/catalog.md b/docs/resources/catalog.md index b070fa36bc..3e0434edc1 100644 --- a/docs/resources/catalog.md +++ b/docs/resources/catalog.md @@ -28,6 +28,7 @@ The following arguments are required: * `storage_root` - (Optional) Managed location of the catalog. Location in cloud storage where data for managed tables will be stored. If not specified, the location will default to the metastore root location. Change forces creation of a new resource. * `provider_name` - (Optional) For Delta Sharing Catalogs: the name of the delta sharing provider. Change forces creation of a new resource. * `share_name` - (Optional) For Delta Sharing Catalogs: the name of the share under the share provider. Change forces creation of a new resource. +* `connection_name` - (Optional) For Foreign Catalogs: the name of the connection to an external data source. Changes forces creation of a new resource. * `owner` - (Optional) Username/groupname/sp application_id of the catalog owner. * `isolation_mode` - (Optional) Whether the catalog is accessible from all workspaces or a specific set of workspaces. Can be `ISOLATED` or `OPEN`. Setting the catalog to `ISOLATED` will automatically allow access from the current workspace. * `comment` - (Optional) User-supplied free-form text. diff --git a/docs/resources/connection.md b/docs/resources/connection.md new file mode 100644 index 0000000000..b19f31780e --- /dev/null +++ b/docs/resources/connection.md @@ -0,0 +1,49 @@ +--- +subcategory: "Unity Catalog" +--- +# databricks_connection (Resource) + +Lakehouse Federation is the query federation platform for Databricks. Databricks uses Unity Catalog to manage query federation. To make a dataset available for read-only querying using Lakehouse Federation, you create the following: + +- A connection, a securable object in Unity Catalog that specifies a path and credentials for accessing an external database system. +- A foreign [catalog](catalog.md) + +This resource manages connections in Unity Catalog + +## Example Usage + +```hcl +resource "databricks_connection" "mysql" { + name = "mysql_connection" + connection_type = "MYSQL" + comment = "this is a connection to mysql db" + options = { + host = "test.mysql.database.azure.com" + port = "3306" + user = "user" + password = "password" + } + properties = { + purpose = "testing" + } +} +``` + +## Argument Reference + +The following arguments are supported: + +- `name` - Name of the Connection. +- `connection_type` - Connection type. `MYSQL` `POSTGRESQL` `SNOWFLAKE` `REDSHIFT` `SQLDW` `SQLSERVER` or `DATABRICKS` are supported. [Up-to-date list of connection type supported](https://docs.databricks.com/query-federation/index.html#supported-data-sources) +- `options` - The key value of options required by the connection, e.g. `host`, `port`, `user` and `password`. +- `owner` - (Optional) Name of the connection owner. +- `properties` - (Optional) Free-form connection properties. +- `comment` - (Optional) Free-form text. + +## Import + +This resource can be imported by `name` + +```bash +terraform import databricks_connection.this +``` diff --git a/internal/acceptance/connection_test.go b/internal/acceptance/connection_test.go new file mode 100644 index 0000000000..320b64a4c2 --- /dev/null +++ b/internal/acceptance/connection_test.go @@ -0,0 +1,22 @@ +package acceptance + +import ( + "testing" +) + +func TestUcAccConnectionsResourceFullLifecycle(t *testing.T) { + unityWorkspaceLevel(t, step{ + Template: ` + resource "databricks_connection" "this" { + name = "name-{var.STICKY_RANDOM}" + connection_type = "MYSQL" + comment = "this is a connection to mysql db" + options = { + host = "test.mysql.database.azure.com" + port = "3306" + user = "user" + password = "password" + } + }`, + }) +} diff --git a/provider/provider.go b/provider/provider.go index 0fd95cd5ad..105787ed9c 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -91,6 +91,7 @@ func DatabricksProvider() *schema.Provider { "databricks_azure_blob_mount": storage.ResourceAzureBlobMount(), "databricks_catalog": catalog.ResourceCatalog(), "databricks_catalog_workspace_binding": catalog.ResourceCatalogWorkspaceBinding(), + "databricks_connection": catalog.ResourceConnection(), "databricks_cluster": clusters.ResourceCluster(), "databricks_cluster_policy": policies.ResourceClusterPolicy(), "databricks_dbfs_file": storage.ResourceDbfsFile(),