Skip to content

Commit

Permalink
Add more attributes to databricks_instance_pool (#1463)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexott
alexott authored Jul 20, 2022
1 parent 9b9c1fa commit e537f4b
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 36 deletions.
2 changes: 1 addition & 1 deletion clusters/clusters_api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -944,7 +944,7 @@ func TestListSparkVersionsWithError(t *testing.T) {
ctx := context.Background()
_, err = NewClustersAPI(ctx, client).ListSparkVersions()
require.Error(t, err)
require.Equal(t, true, strings.Contains(err.Error(), "Invalid JSON received"))
require.Equal(t, true, strings.Contains(err.Error(), "invalid character 'g' looking"))
}

func TestGetLatestSparkVersion(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion clusters/data_spark_version_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,5 +232,5 @@ func TestSparkVersionErrorBadAnswer(t *testing.T) {
ID: ".",
}.Apply(t)
assert.Error(t, err)
require.Equal(t, true, strings.Contains(err.Error(), "Invalid JSON received"))
require.Equal(t, true, strings.Contains(err.Error(), "invalid character 'g' looking"))
}
3 changes: 1 addition & 2 deletions common/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,7 @@ func (c *DatabricksClient) unmarshall(path string, body []byte, response any) er
ErrorCode: "UNKNOWN",
StatusCode: 200,
Resource: "..." + path,
Message: fmt.Sprintf("Invalid JSON received (%d bytes): %v",
len(body), string(body)),
Message: fmt.Sprintf("%s\n%s", err.Error(), onlyNBytes(string(body), 32)),
}
}

Expand Down
14 changes: 13 additions & 1 deletion common/http_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ func TestPost_Error(t *testing.T) {
ScimDetail: "some",
}, &resp)
require.Error(t, err)
assert.True(t, strings.HasPrefix(err.Error(), "Invalid JSON received (16 bytes)"),
assert.True(t, strings.HasPrefix(err.Error(), "invalid character 'c'"),
"Actual message: %s", err.Error())
}

Expand Down Expand Up @@ -311,6 +311,18 @@ func TestUnmarshall(t *testing.T) {
require.NoError(t, err)
}

func TestUnmarshallError(t *testing.T) {
v := struct {
Correct string `json:"c"`
Incorrect int `json:"i"`
}{}
ws := DatabricksClient{}
err := ws.unmarshall("/a/b/c", []byte(`{"c":"val", "i":"123"}`), &v)
require.Error(t, err)
require.EqualError(t, err,
"json: cannot unmarshal string into Go struct field .i of type int\n{\"c\":\"val\", \"i\":\"123\"}")
}

func TestAPI2(t *testing.T) {
ws := DatabricksClient{Host: "ht_tp://example.com/"}
err := ws.completeUrl(&http.Request{})
Expand Down
98 changes: 67 additions & 31 deletions pools/resource_instance_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,22 +43,48 @@ type InstancePoolDiskSpec struct {
DiskSize int32 `json:"disk_size,omitempty"`
}

type AwsAllocationStrategy string

const (
// AwsAllocationStrategyLowestPrice is allocation type for AWS Instance fleets
AwsAllocationStrategyLowestPrice = "LOWEST_PRICE"
// AwsAllocationStrategyCapacityOptimized is allocation type for AWS Instance fleets
AwsAllocationStrategyCapacityOptimized = "CAPACITY_OPTIMIZED"
)

type AwsFleetOption struct {
AllocationStrategy AwsAllocationStrategy `json:"allocation_strategy" tf:"force_new,suppress_diff"`
InstancePoolsToUseCount int32 `json:"instance_pools_to_use_count,omitempty" tf:"suppress_diff"`
}

type AwsFleetLaunchTemplateOverride struct {
AvailabilityZone string `json:"availability_zone" tf:"force_new,suppress_diff"`
InstanceType string `json:"instance_type" tf:"force_new,suppress_diff"`
}

type AwsInstancePoolFleetAttributes struct {
FleetSpotOption *AwsFleetOption `json:"fleet_spot_option,omitempty" tf:"force_new,suppress_diff,conflicts:fleet_on_demand_option"`
FleetOnDemandOption *AwsFleetOption `json:"fleet_on_demand_option,omitempty" tf:"force_new,suppress_diff,conflicts:fleet_spot_option"`
FleetLaunchTemplateOverride []AwsFleetLaunchTemplateOverride `json:"launch_template_overrides" tf:"suppress_diff,force_new,slice_set,alias:launch_template_override"`
}

// InstancePool describes the instance pool object on Databricks
type InstancePool struct {
InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
InstancePoolName string `json:"instance_pool_name"`
MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
MaxCapacity int32 `json:"max_capacity,omitempty" tf:"suppress_diff"`
IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty" tf:"force_new,suppress_diff"`
AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty" tf:"force_new,suppress_diff"`
GcpAttributes *InstancePoolGcpAttributes `json:"gcp_attributes,omitempty" tf:"force_new,suppress_diff"`
NodeTypeID string `json:"node_type_id" tf:"force_new"`
CustomTags map[string]string `json:"custom_tags,omitempty" tf:"force_new"`
EnableElasticDisk bool `json:"enable_elastic_disk,omitempty" tf:"force_new,suppress_diff"`
DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty" tf:"force_new"`
PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty" tf:"force_new"`
PreloadedDockerImages []clusters.DockerImage `json:"preloaded_docker_images,omitempty" tf:"force_new,slice_set,alias:preloaded_docker_image"`
InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
InstancePoolName string `json:"instance_pool_name"`
MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
MaxCapacity int32 `json:"max_capacity,omitempty" tf:"suppress_diff"`
IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty" tf:"force_new,suppress_diff,computed"`
AwsInstancePoolFleetAttributes *AwsInstancePoolFleetAttributes `json:"instance_pool_fleet_attributes,omitempty" tf:"force_new,suppress_diff,conflicts:node_type_id"`
AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty" tf:"force_new,suppress_diff"`
GcpAttributes *InstancePoolGcpAttributes `json:"gcp_attributes,omitempty" tf:"force_new,suppress_diff"`
NodeTypeID string `json:"node_type_id,omitempty" tf:"suppress_diff,force_new,conflicts:instance_pool_fleet_attributes"`
CustomTags map[string]string `json:"custom_tags,omitempty" tf:"force_new"`
EnableElasticDisk bool `json:"enable_elastic_disk,omitempty" tf:"force_new,suppress_diff"`
DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty" tf:"force_new"`
PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty" tf:"force_new"`
PreloadedDockerImages []clusters.DockerImage `json:"preloaded_docker_images,omitempty" tf:"force_new,slice_set,alias:preloaded_docker_image"`
}

// InstancePoolStats contains the stats on a given pool
Expand All @@ -71,23 +97,24 @@ type InstancePoolStats struct {

// InstancePoolAndStats encapsulates a get response from the GET api for instance pools on Databricks
type InstancePoolAndStats struct {
InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
InstancePoolName string `json:"instance_pool_name"`
MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
MaxCapacity int32 `json:"max_capacity,omitempty"`
AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"`
AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty"`
GcpAttributes *InstancePoolGcpAttributes `json:"gcp_attributes,omitempty"`
NodeTypeID string `json:"node_type_id"`
DefaultTags map[string]string `json:"default_tags,omitempty" tf:"computed"`
CustomTags map[string]string `json:"custom_tags,omitempty"`
IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"`
PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"`
State string `json:"state,omitempty"`
Stats *InstancePoolStats `json:"stats,omitempty"`
PreloadedDockerImages []clusters.DockerImage `json:"preloaded_docker_images,omitempty" tf:"slice_set,alias:preloaded_docker_image"`
InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
InstancePoolName string `json:"instance_pool_name"`
MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
MaxCapacity int32 `json:"max_capacity,omitempty"`
AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"`
AwsInstancePoolFleetAttributes []AwsInstancePoolFleetAttributes `json:"instance_pool_fleet_attributes,omitempty"`
AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty"`
GcpAttributes *InstancePoolGcpAttributes `json:"gcp_attributes,omitempty"`
NodeTypeID string `json:"node_type_id,omitempty"`
DefaultTags map[string]string `json:"default_tags,omitempty" tf:"computed"`
CustomTags map[string]string `json:"custom_tags,omitempty"`
IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"`
PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"`
State string `json:"state,omitempty"`
Stats *InstancePoolStats `json:"stats,omitempty"`
PreloadedDockerImages []clusters.DockerImage `json:"preloaded_docker_images,omitempty" tf:"slice_set,alias:preloaded_docker_image"`
}

// InstancePoolList shows list of instance pools
Expand Down Expand Up @@ -185,6 +212,15 @@ func ResourceInstancePool() *schema.Resource {
clusters.EbsVolumeTypeThroughputOptimizedHdd,
}, false)
}
if v, err := common.SchemaPath(s, "instance_pool_fleet_attributes", "fleet_on_demand_option", "allocation_strategy"); err == nil {
v.ValidateFunc = validation.StringInSlice([]string{AwsAllocationStrategyLowestPrice}, false)
}
if v, err := common.SchemaPath(s, "instance_pool_fleet_attributes", "fleet_spot_option", "allocation_strategy"); err == nil {
v.ValidateFunc = validation.StringInSlice([]string{
AwsAllocationStrategyLowestPrice,
AwsAllocationStrategyCapacityOptimized,
}, false)
}
if v, err := common.SchemaPath(s, "preloaded_docker_image", "url"); err == nil {
v.ForceNew = true
}
Expand Down

0 comments on commit e537f4b

Please sign in to comment.