From 5adb2ddadb041aa867e14b41bb7ba228b2e8f0d6 Mon Sep 17 00:00:00 2001 From: Alex Ott Date: Tue, 23 Mar 2021 10:05:38 +0100 Subject: [PATCH] Azure spot instances in the instance pool --- CHANGELOG.md | 2 +- compute/model.go | 62 ++++++++++++++++++------------- compute/resource_instance_pool.go | 8 ++++ docs/resources/cluster.md | 6 +-- docs/resources/instance_pool.md | 11 ++++++ 5 files changed, 59 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41359a8d8d..a0aedb8b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 0.3.2 * Fix incorrect escaping of notebook names ([#566](https://github.com/databrickslabs/terraform-provider-databricks/pull/566)) -* Added support for spot instances on Azure (TODO) +* Added support for spot instances on Azure ([#571](https://github.com/databrickslabs/terraform-provider-databricks/pull/571)) ## 0.3.1 diff --git a/compute/model.go b/compute/model.go index ba431c4a73..b70cb571c2 100644 --- a/compute/model.go +++ b/compute/model.go @@ -24,6 +24,7 @@ const ( AwsAvailabilitySpotWithFallback = "SPOT_WITH_FALLBACK" ) +// https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureavailability const ( // AzureAvailabilitySpot is spot instance type for clusters AzureAvailabilitySpot = "SPOT_AZURE" @@ -133,7 +134,7 @@ type AwsAttributes struct { } // AzureAttributes encapsulates the Azure attributes for Azure based clusters -// TODO: add link to documentation after release +// https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#clusterazureattributes type AzureAttributes struct { FirstOnDemand int32 `json:"first_on_demand,omitempty" tf:"computed"` Availability Availability `json:"availability,omitempty" tf:"computed"` @@ -378,6 +379,13 @@ type InstancePoolAwsAttributes struct { SpotBidPricePercent int32 `json:"spot_bid_price_percent,omitempty"` } +// InstancePoolAzureAttributes contains aws attributes for Azure Databricks deployments for instance pools +// https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/instance-pools#clusterinstancepoolazureattributes +type InstancePoolAzureAttributes struct { + Availability Availability `json:"availability,omitempty"` + SpotBidMaxPrice float64 `json:"spot_bid_max_price,omitempty" tf:"computed"` +} + // InstancePoolDiskType contains disk type information for each of the different cloud service providers type InstancePoolDiskType struct { AzureDiskVolumeType string `json:"azure_disk_volume_type,omitempty"` @@ -393,17 +401,18 @@ type InstancePoolDiskSpec struct { // InstancePool describes the instance pool object on Databricks type InstancePool struct { - InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"` - InstancePoolName string `json:"instance_pool_name"` - MinIdleInstances int32 `json:"min_idle_instances,omitempty"` - MaxCapacity int32 `json:"max_capacity,omitempty"` - IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"` - AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"` - NodeTypeID string `json:"node_type_id"` - CustomTags map[string]string `json:"custom_tags,omitempty"` - EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` - DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"` - PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"` + InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"` + InstancePoolName string `json:"instance_pool_name"` + MinIdleInstances int32 `json:"min_idle_instances,omitempty"` + MaxCapacity int32 `json:"max_capacity,omitempty"` + IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"` + AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"` + AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty"` + NodeTypeID string `json:"node_type_id"` + CustomTags map[string]string `json:"custom_tags,omitempty"` + EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` + DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"` + PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"` } // InstancePoolStats contains the stats on a given pool @@ -416,20 +425,21 @@ type InstancePoolStats struct { // InstancePoolAndStats encapsulates a get response from the GET api for instance pools on Databricks type InstancePoolAndStats struct { - InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"` - InstancePoolName string `json:"instance_pool_name"` - MinIdleInstances int32 `json:"min_idle_instances,omitempty"` - MaxCapacity int32 `json:"max_capacity,omitempty"` - AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"` - NodeTypeID string `json:"node_type_id"` - DefaultTags map[string]string `json:"default_tags,omitempty" tf:"computed"` - CustomTags map[string]string `json:"custom_tags,omitempty"` - IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"` - EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` - DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"` - PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"` - State string `json:"state,omitempty"` - Stats *InstancePoolStats `json:"stats,omitempty"` + InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"` + InstancePoolName string `json:"instance_pool_name"` + MinIdleInstances int32 `json:"min_idle_instances,omitempty"` + MaxCapacity int32 `json:"max_capacity,omitempty"` + AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"` + AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty"` + NodeTypeID string `json:"node_type_id"` + DefaultTags map[string]string `json:"default_tags,omitempty" tf:"computed"` + CustomTags map[string]string `json:"custom_tags,omitempty"` + IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"` + EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` + DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"` + PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"` + State string `json:"state,omitempty"` + Stats *InstancePoolStats `json:"stats,omitempty"` } // InstancePoolList shows list of instance pools diff --git a/compute/resource_instance_pool.go b/compute/resource_instance_pool.go index 1f09ff43c1..382be51631 100644 --- a/compute/resource_instance_pool.go +++ b/compute/resource_instance_pool.go @@ -61,6 +61,8 @@ func ResourceInstancePool() *schema.Resource { s["custom_tags"].ForceNew = true s["enable_elastic_disk"].ForceNew = true s["enable_elastic_disk"].Default = true + s["aws_attributes"].ConflictsWith = []string{"azure_attributes"} + s["azure_attributes"].ConflictsWith = []string{"aws_attributes"} // TODO: check if it's really force new... if v, err := common.SchemaPath(s, "aws_attributes", "availability"); err == nil { v.ForceNew = true @@ -71,6 +73,12 @@ func ResourceInstancePool() *schema.Resource { if v, err := common.SchemaPath(s, "aws_attributes", "spot_bid_price_percent"); err == nil { v.ForceNew = true } + if v, err := common.SchemaPath(s, "azure_attributes", "availability"); err == nil { + v.ForceNew = true + } + if v, err := common.SchemaPath(s, "azure_attributes", "spot_bid_max_price"); err == nil { + v.ForceNew = true + } if v, err := common.SchemaPath(s, "disk_spec", "disk_type", "azure_disk_volume_type"); err == nil { v.ForceNew = true // nolint diff --git a/docs/resources/cluster.md b/docs/resources/cluster.md index 22583859f1..607866b7c5 100644 --- a/docs/resources/cluster.md +++ b/docs/resources/cluster.md @@ -269,7 +269,7 @@ The following options are available: ## azure_attributes -`azure_attributes` optional configuration block contains attributes related to [clusters running on Azure](TODO). +`azure_attributes` optional configuration block contains attributes related to [clusters running on Azure](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes). -> **Note** *(Azure only)* Please specify empty configuration block (`azure_attributes {}`), even if you're not setting any custom values. This will prevent any resource update issues. @@ -293,11 +293,11 @@ resource "databricks_cluster" "this" { } ``` -The following options are available: +The following options are [available](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes): * `availability` - (Optional) Availability type used for all subsequent nodes past the `first_on_demand` ones. Valid values are `SPOT_AZURE`, `SPOT_WITH_FALLBACK`, and `ON_DEMAND_AZURE`. Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. * `first_on_demand` - (Optional) The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances, and the remainder will be placed on availability instances. This value does not affect cluster size and cannot be mutated over the lifetime of a cluster. -* `spot_bid_max_price` - (Optional) The max price for Azure spot instances. +* `spot_bid_max_price` - (Optional) The max price for Azure spot instances. Use `-1` to specify lowest price. ## gcp_attributes diff --git a/docs/resources/instance_pool.md b/docs/resources/instance_pool.md index e4b2d5dc3d..b4d4f9198b 100644 --- a/docs/resources/instance_pool.md +++ b/docs/resources/instance_pool.md @@ -56,6 +56,17 @@ The following arguments are required: * `availability` - (Optional) (String) Availability type used for all instances in the pool. Only `ON_DEMAND` and `SPOT` are supported. * `zone_id` - (Required) (String) Identifier for the availability zone/datacenter in which the instance pool resides. This string is of a form like `"us-west-2a"`. The provided availability zone must be in the same region as the Databricks deployment. For example, `"us-west-2a"` is not a valid zone ID if the Databricks deployment resides in the `"us-east-1"` region. This is an optional field. If not specified, a default zone is used. You can find the list of available zones as well as the default value by using the [List Zones API](https://docs.databricks.com/dev-tools/api/latest/clusters.html#clusterclusterservicelistavailablezones). +## azure_attributes Configuration Block + +`azure_attributes` optional configuration block contains attributes related to [instance pools on Azure](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/instance-pools#--instancepoolazureattributes). + +-> **Note** *(Azure only)* Please specify empty configuration block (`azure_attributes {}`), even if you're not setting any custom values. This will prevent any resource update issues. + +The following options are [available](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes): + +* `availability` - (Optional) Availability type used for all subsequent nodes past the `first_on_demand` ones. Valid values are `SPOT_AZURE` and `ON_DEMAND_AZURE`. +* `spot_bid_max_price` - (Optional) The max price for Azure spot instances. Use `-1` to specify lowest price. + ### disk_spec Configuration Block