Azure spot instances in the instance pool

databricks · Mar 23, 2021 · 5adb2dd · 5adb2dd
1 parent 63494ff
commit 5adb2dd
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 30 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,7 +3,7 @@
 ## 0.3.2
 
 * Fix incorrect escaping of notebook names ([#566](https://github.com/databrickslabs/terraform-provider-databricks/pull/566))
-* Added support for spot instances on Azure (TODO)
+* Added support for spot instances on Azure ([#571](https://github.com/databrickslabs/terraform-provider-databricks/pull/571))
 
 ## 0.3.1
 

diff --git a/compute/model.go b/compute/model.go
@@ -24,6 +24,7 @@ const (
  AwsAvailabilitySpotWithFallback = "SPOT_WITH_FALLBACK"
 )
 
+// https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureavailability
 const (
  // AzureAvailabilitySpot is spot instance type for clusters
  AzureAvailabilitySpot = "SPOT_AZURE"
@@ -133,7 +134,7 @@ type AwsAttributes struct {
 }
 
 // AzureAttributes encapsulates the Azure attributes for Azure based clusters
-// TODO: add link to documentation after release
+// https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#clusterazureattributes
 type AzureAttributes struct {
  FirstOnDemand int32 `json:"first_on_demand,omitempty" tf:"computed"`
  Availability Availability `json:"availability,omitempty" tf:"computed"`
@@ -378,6 +379,13 @@ type InstancePoolAwsAttributes struct {
  SpotBidPricePercent int32 `json:"spot_bid_price_percent,omitempty"`
 }
 
+// InstancePoolAzureAttributes contains aws attributes for Azure Databricks deployments for instance pools
+// https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/instance-pools#clusterinstancepoolazureattributes
+type InstancePoolAzureAttributes struct {
+ Availability Availability `json:"availability,omitempty"`
+ SpotBidMaxPrice float64 `json:"spot_bid_max_price,omitempty" tf:"computed"`
+}
+
 // InstancePoolDiskType contains disk type information for each of the different cloud service providers
 type InstancePoolDiskType struct {
  AzureDiskVolumeType string `json:"azure_disk_volume_type,omitempty"`
@@ -393,17 +401,18 @@ type InstancePoolDiskSpec struct {
 
 // InstancePool describes the instance pool object on Databricks
 type InstancePool struct {
- InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
- InstancePoolName string `json:"instance_pool_name"`
- MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
- MaxCapacity int32 `json:"max_capacity,omitempty"`
- IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
- AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"`
- NodeTypeID string `json:"node_type_id"`
- CustomTags map[string]string `json:"custom_tags,omitempty"`
- EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
- DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"`
- PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"`
+ InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
+ InstancePoolName string `json:"instance_pool_name"`
+ MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
+ MaxCapacity int32 `json:"max_capacity,omitempty"`
+ IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
+ AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"`
+ AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty"`
+ NodeTypeID string `json:"node_type_id"`
+ CustomTags map[string]string `json:"custom_tags,omitempty"`
+ EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
+ DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"`
+ PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"`
 }
 
 // InstancePoolStats contains the stats on a given pool
@@ -416,20 +425,21 @@ type InstancePoolStats struct {
 
 // InstancePoolAndStats encapsulates a get response from the GET api for instance pools on Databricks
 type InstancePoolAndStats struct {
- InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
- InstancePoolName string `json:"instance_pool_name"`
- MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
- MaxCapacity int32 `json:"max_capacity,omitempty"`
- AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"`
- NodeTypeID string `json:"node_type_id"`
- DefaultTags map[string]string `json:"default_tags,omitempty" tf:"computed"`
- CustomTags map[string]string `json:"custom_tags,omitempty"`
- IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
- EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
- DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"`
- PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"`
- State string `json:"state,omitempty"`
- Stats *InstancePoolStats `json:"stats,omitempty"`
+ InstancePoolID string `json:"instance_pool_id,omitempty" tf:"computed"`
+ InstancePoolName string `json:"instance_pool_name"`
+ MinIdleInstances int32 `json:"min_idle_instances,omitempty"`
+ MaxCapacity int32 `json:"max_capacity,omitempty"`
+ AwsAttributes *InstancePoolAwsAttributes `json:"aws_attributes,omitempty"`
+ AzureAttributes *InstancePoolAzureAttributes `json:"azure_attributes,omitempty"`
+ NodeTypeID string `json:"node_type_id"`
+ DefaultTags map[string]string `json:"default_tags,omitempty" tf:"computed"`
+ CustomTags map[string]string `json:"custom_tags,omitempty"`
+ IdleInstanceAutoTerminationMinutes int32 `json:"idle_instance_autotermination_minutes"`
+ EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"`
+ DiskSpec *InstancePoolDiskSpec `json:"disk_spec,omitempty"`
+ PreloadedSparkVersions []string `json:"preloaded_spark_versions,omitempty"`
+ State string `json:"state,omitempty"`
+ Stats *InstancePoolStats `json:"stats,omitempty"`
 }
 
 // InstancePoolList shows list of instance pools

diff --git a/compute/resource_instance_pool.go b/compute/resource_instance_pool.go
@@ -61,6 +61,8 @@ func ResourceInstancePool() *schema.Resource {
  s["custom_tags"].ForceNew = true
  s["enable_elastic_disk"].ForceNew = true
  s["enable_elastic_disk"].Default = true
+ s["aws_attributes"].ConflictsWith = []string{"azure_attributes"}
+ s["azure_attributes"].ConflictsWith = []string{"aws_attributes"}
  // TODO: check if it's really force new...
  if v, err := common.SchemaPath(s, "aws_attributes", "availability"); err == nil {
  v.ForceNew = true
@@ -71,6 +73,12 @@ func ResourceInstancePool() *schema.Resource {
  if v, err := common.SchemaPath(s, "aws_attributes", "spot_bid_price_percent"); err == nil {
  v.ForceNew = true
  }
+ if v, err := common.SchemaPath(s, "azure_attributes", "availability"); err == nil {
+ v.ForceNew = true
+ }
+ if v, err := common.SchemaPath(s, "azure_attributes", "spot_bid_max_price"); err == nil {
+ v.ForceNew = true
+ }
  if v, err := common.SchemaPath(s, "disk_spec", "disk_type", "azure_disk_volume_type"); err == nil {
  v.ForceNew = true
  // nolint

diff --git a/docs/resources/cluster.md b/docs/resources/cluster.md
@@ -269,7 +269,7 @@ The following options are available:
 
 ## azure_attributes
 
-`azure_attributes` optional configuration block contains attributes related to [clusters running on Azure](TODO).
+`azure_attributes` optional configuration block contains attributes related to [clusters running on Azure](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes).
 
 -> **Note** *(Azure only)* Please specify empty configuration block (`azure_attributes {}`), even if you're not setting any custom values. This will prevent any resource update issues.
 
@@ -293,11 +293,11 @@ resource "databricks_cluster" "this" {
 }
 ```
 
-The following options are available:
+The following options are [available](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes):
 
 * `availability` - (Optional) Availability type used for all subsequent nodes past the `first_on_demand` ones. Valid values are `SPOT_AZURE`, `SPOT_WITH_FALLBACK`, and `ON_DEMAND_AZURE`. Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster.
 * `first_on_demand` - (Optional) The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances, and the remainder will be placed on availability instances. This value does not affect cluster size and cannot be mutated over the lifetime of a cluster.
-* `spot_bid_max_price` - (Optional) The max price for Azure spot instances.
+* `spot_bid_max_price` - (Optional) The max price for Azure spot instances. Use `-1` to specify lowest price.
 
 ## gcp_attributes
 

diff --git a/docs/resources/instance_pool.md b/docs/resources/instance_pool.md
@@ -56,6 +56,17 @@ The following arguments are required:
 * `availability` - (Optional) (String) Availability type used for all instances in the pool. Only `ON_DEMAND` and `SPOT` are supported.
 * `zone_id` - (Required) (String) Identifier for the availability zone/datacenter in which the instance pool resides. This string is of a form like `"us-west-2a"`. The provided availability zone must be in the same region as the Databricks deployment. For example, `"us-west-2a"` is not a valid zone ID if the Databricks deployment resides in the `"us-east-1"` region. This is an optional field. If not specified, a default zone is used. You can find the list of available zones as well as the default value by using the [List Zones API](https://docs.databricks.com/dev-tools/api/latest/clusters.html#clusterclusterservicelistavailablezones).
 
+## azure_attributes Configuration Block
+
+`azure_attributes` optional configuration block contains attributes related to [instance pools on Azure](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/instance-pools#--instancepoolazureattributes).
+
+-> **Note** *(Azure only)* Please specify empty configuration block (`azure_attributes {}`), even if you're not setting any custom values. This will prevent any resource update issues.
+
+The following options are [available](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes):
+
+* `availability` - (Optional) Availability type used for all subsequent nodes past the `first_on_demand` ones. Valid values are `SPOT_AZURE` and `ON_DEMAND_AZURE`.
+* `spot_bid_max_price` - (Optional) The max price for Azure spot instances. Use `-1` to specify lowest price.
+
 
 ### disk_spec Configuration Block