Skip to content

Commit

Permalink
Added delta_sharing_* support to databricks_metastore (#1334)
Browse files Browse the repository at this point in the history 
* added support delta sharing enablement via metastore resource
* added new optional fields to metastore docs
* added delta_sharing_organization_name and fixed logic for setting delta_sharing_enabled to true which always requires the delta_sharing_recipient_token_lifetime_in_seconds to be passed
* updated:
 1. DeltaSharingRecipientTokenLifetimeInSeconds to int64
 2. DeltaSharingRecipientTokenLifetimeInSeconds default to 90 days same as our tokens
 3. separated create metastore test from create metastore with delta sharing
* removed defaults simplified the logic and both tests now pass as expected
  • Loading branch information
@stikkireddy
stikkireddy authored Jun 2, 2022
1 parent 2b3f764 commit ddd9fe6
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 7 deletions.
35 changes: 28 additions & 7 deletions catalog/resource_metastore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,22 @@ func NewMetastoresAPI(ctx context.Context, m interface{}) MetastoresAPI {
}

type MetastoreInfo struct {
Name string `json:"name"`
StorageRoot string `json:"storage_root" tf:"force_new"`
DefaultDacID string `json:"default_data_access_config_id,omitempty"`
Owner string `json:"owner,omitempty" tf:"computed"`
MetastoreID string `json:"metastore_id,omitempty" tf:"computed"`
WorkspaceIDs []int64 `json:"workspace_ids,omitempty" tf:"computed"`
Name string `json:"name"`
StorageRoot string `json:"storage_root" tf:"force_new"`
DefaultDacID string `json:"default_data_access_config_id,omitempty"`
Owner string `json:"owner,omitempty" tf:"computed"`
MetastoreID string `json:"metastore_id,omitempty" tf:"computed"`
WorkspaceIDs []int64 `json:"workspace_ids,omitempty" tf:"computed"`
Region string `json:"region,omitempty" tf:"computed"`
Cloud string `json:"cloud,omitempty" tf:"computed"`
GlobalMetastoreId string `json:"global_metastore_id,omitempty" tf:"computed"`
CreatedAt int64 `json:"created_at,omitempty" tf:"computed"`
CreatedBy string `json:"created_by,omitempty" tf:"computed"`
UpdatedAt int64 `json:"updated_at,omitempty" tf:"computed"`
UpdatedBy string `json:"updated_by,omitempty" tf:"computed"`
DeltaSharingEnabled bool `json:"delta_sharing_enabled,omitempty"`
DeltaSharingRecipientTokenLifetimeInSeconds int64 `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"`
DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"`
}

type CreateMetastore struct {
Expand Down Expand Up @@ -66,6 +76,8 @@ func ResourceMetastore() *schema.Resource {
Type: schema.TypeBool,
Optional: true,
}
m["delta_sharing_enabled"].RequiredWith = []string{"delta_sharing_recipient_token_lifetime_in_seconds"}
m["delta_sharing_recipient_token_lifetime_in_seconds"].RequiredWith = []string{"delta_sharing_enabled"}
m["storage_root"].DiffSuppressFunc = func(k, old, new string, d *schema.ResourceData) bool {
if strings.HasPrefix(old, new) {
log.Printf("[DEBUG] Ignoring configuration drift from %s to %s", old, new)
Expand All @@ -77,7 +89,8 @@ func ResourceMetastore() *schema.Resource {
})
update := func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
// other fields to come later
updatable := []string{"owner", "name"}
updatable := []string{"owner", "name", "delta_sharing_enabled",
"delta_sharing_recipient_token_lifetime_in_seconds", "delta_sharing_organization_name"}
patch := map[string]interface{}{}
for _, field := range updatable {
old, new := d.GetChange(field)
Expand All @@ -87,6 +100,14 @@ func ResourceMetastore() *schema.Resource {
if field == "name" && old == "" {
continue
}
// delta sharing enabled and new is true must always be accompanied by a value for
// delta_sharing_recipient_token_lifetime_in_seconds
if field == "delta_sharing_enabled" && old != new && new == true &&
!d.HasChange("delta_sharing_recipient_token_lifetime_in_seconds") {
patch["delta_sharing_recipient_token_lifetime_in_seconds"] =
d.Get("delta_sharing_recipient_token_lifetime_in_seconds")
}

patch[field] = new
}
if len(patch) == 0 {
Expand Down
120 changes: 120 additions & 0 deletions catalog/resource_metastore_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,52 @@ func TestCreateMetastore(t *testing.T) {
}.ApplyNoError(t)
}

func TestCreateMetastore_DeltaSharing(t *testing.T) {
qa.ResourceFixture{
Fixtures: []qa.HTTPFixture{
{
Method: "POST",
Resource: "/api/2.0/unity-catalog/metastores",
ExpectedRequest: MetastoreInfo{
StorageRoot: "s3://b",
Name: "a",
},
Response: MetastoreInfo{
MetastoreID: "abc",
},
},
{
Method: "GET",
Resource: "/api/2.0/unity-catalog/metastores/abc",
Response: MetastoreInfo{
StorageRoot: "s3://b/abc",
Name: "a",
},
},
{
Method: "PATCH",
Resource: "/api/2.0/unity-catalog/metastores/abc",
ExpectedRequest: map[string]interface{}{
"owner": "administrators",
"delta_sharing_enabled": true,
"delta_sharing_recipient_token_lifetime_in_seconds": 0,
"delta_sharing_organization_name": "acme",
},
},
},
Resource: ResourceMetastore(),
Create: true,
HCL: `
name = "a"
storage_root = "s3://b"
owner = "administrators"
delta_sharing_enabled = true
delta_sharing_recipient_token_lifetime_in_seconds = 0
delta_sharing_organization_name = "acme"
`,
}.ApplyNoError(t)
}

func TestDeleteMetastore(t *testing.T) {
qa.ResourceFixture{
Fixtures: []qa.HTTPFixture{
Expand Down Expand Up @@ -93,3 +139,77 @@ func TestForceDeleteMetastore(t *testing.T) {
`,
}.ApplyNoError(t)
}

func TestUpdateMetastore_NoChanges(t *testing.T) {
qa.ResourceFixture{
Fixtures: []qa.HTTPFixture{
{
Method: "GET",
Resource: "/api/2.0/unity-catalog/metastores/abc",
Response: MetastoreInfo{
StorageRoot: "s3://b/abc",
Name: "a",
},
},
},
Resource: ResourceMetastore(),
ID: "abc",
Update: true,
RequiresNew: true,
InstanceState: map[string]string{
"name": "abc",
"storage_root": "s3:/a",
"owner": "admin",
"delta_sharing_enabled": "true",
"delta_sharing_recipient_token_lifetime_in_seconds": "1002",
},
HCL: `
name = "abc"
storage_root = "s3:/a"
owner = "admin"
delta_sharing_enabled = true
delta_sharing_recipient_token_lifetime_in_seconds = 1002
`,
}.ApplyNoError(t)
}

func TestUpdateMetastore_DeltaSharingEnableOnly(t *testing.T) {
qa.ResourceFixture{
Fixtures: []qa.HTTPFixture{
{
Method: "PATCH",
Resource: "/api/2.0/unity-catalog/metastores/abc",
ExpectedRequest: map[string]interface{}{
"delta_sharing_enabled": true,
"delta_sharing_recipient_token_lifetime_in_seconds": 1002,
},
},
{
Method: "GET",
Resource: "/api/2.0/unity-catalog/metastores/abc",
Response: MetastoreInfo{
StorageRoot: "s3://b/abc",
Name: "a",
},
},
},
Resource: ResourceMetastore(),
ID: "abc",
Update: true,
RequiresNew: true,
InstanceState: map[string]string{
"name": "abc",
"storage_root": "s3:/a",
"owner": "admin",
"delta_sharing_enabled": "false",
"delta_sharing_recipient_token_lifetime_in_seconds": "1002",
},
HCL: `
name = "abc"
storage_root = "s3:/a"
owner = "admin"
delta_sharing_enabled = true
delta_sharing_recipient_token_lifetime_in_seconds = 1002
`,
}.ApplyNoError(t)
}
3 changes: 3 additions & 0 deletions docs/resources/metastore.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ The following arguments are required:
* `name` - Name of metastore.
* `storage_root` - Path on cloud storage account, where managed [databricks_table](table.md) are stored. Change forces creation of a new resource.
* `owner` - (Optional) Username/groupname of Metastore owner.
* `delta_sharing_enabled` - (Optional) Required along with `delta_sharing_recipient_token_lifetime_in_seconds`. Used to enable delta sharing on the metastore.
* `delta_sharing_recipient_token_lifetime_in_seconds` - (Optional) Required along with `delta_sharing_enabled`. Used to set expiration duration in seconds on recipient data access tokens. Set to 0 for unlimited duration.
* `delta_sharing_organization_name` - (Optional) The organization name of a Delta Sharing entity. This field is used for Databricks to Databricks sharing. Once this is set it cannot be removed and can only be modified to another valid value. To delete this value please taint and recreate the resource.
* `force_destroy` - (Optional) Destroy metastore regardless of its contents.

## Import
Expand Down

0 comments on commit ddd9fe6

Please sign in to comment.