Add autotuning_config and cohort fields for Dataproc batch (#12357)…

… (#20410) [upstream:37310e5ad8030ab802f85c34d56da0b8730c0875] Signed-off-by: Modular Magician <magic-modules@google.com>
hashicorp · Nov 20, 2024 · aff6d24 · aff6d24
1 parent 1b6b248
commit aff6d24
Show file tree

Hide file tree

Showing 4 changed files with 209 additions and 0 deletions.
diff --git a/.changelog/12357.txt b/.changelog/12357.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+dataproc: added `autotuning_config` and `cohort` fields to `google_dataproc_batch`
+```
diff --git a/google/services/dataproc/resource_dataproc_batch.go b/google/services/dataproc/resource_dataproc_batch.go
@@ -30,6 +30,7 @@ import (
 
 	"github.com/hashicorp/terraform-provider-google/google/tpgresource"
 	transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"
+	"github.com/hashicorp/terraform-provider-google/google/verify"
 )
 
 /*
@@ -285,6 +286,34 @@ Supported file types: .py, .egg, and .zip.`,
 				MaxItems:    1,
 				Elem: &schema.Resource{
 					Schema: map[string]*schema.Schema{
+						"autotuning_config": {
+							Type:        schema.TypeList,
+							Optional:    true,
+							ForceNew:    true,
+							Description: `Optional. Autotuning configuration of the workload.`,
+							MaxItems:    1,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"scenarios": {
+										Type:        schema.TypeList,
+										Optional:    true,
+										ForceNew:    true,
+										Description: `Optional. Scenarios for which tunings are applied. Possible values: ["SCALING", "BROADCAST_HASH_JOIN", "MEMORY"]`,
+										Elem: &schema.Schema{
+											Type:         schema.TypeString,
+											ValidateFunc: verify.ValidateEnum([]string{"SCALING", "BROADCAST_HASH_JOIN", "MEMORY"}),
+										},
+										RequiredWith: []string{"runtime_config.0.cohort"},
+									},
+								},
+							},
+						},
+						"cohort": {
+							Type:        schema.TypeString,
+							Optional:    true,
+							ForceNew:    true,
+							Description: `Optional. Cohort identifier. Identifies families of the workloads having the same shape, e.g. daily ETL jobs.`,
+						},
 						"container_image": {
 							Type:        schema.TypeString,
 							Optional:    true,
@@ -1132,6 +1161,10 @@ func flattenDataprocBatchRuntimeConfig(v interface{}, d *schema.ResourceData, co
 		flattenDataprocBatchRuntimeConfigProperties(original["properties"], d, config)
 	transformed["effective_properties"] =
 		flattenDataprocBatchRuntimeConfigEffectiveProperties(original["effective_properties"], d, config)
+	transformed["autotuning_config"] =
+		flattenDataprocBatchRuntimeConfigAutotuningConfig(original["autotuningConfig"], d, config)
+	transformed["cohort"] =
+		flattenDataprocBatchRuntimeConfigCohort(original["cohort"], d, config)
 	return []interface{}{transformed}
 }
 func flattenDataprocBatchRuntimeConfigVersion(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
@@ -1150,6 +1183,27 @@ func flattenDataprocBatchRuntimeConfigEffectiveProperties(v interface{}, d *sche
 	return v
 }
 
+func flattenDataprocBatchRuntimeConfigAutotuningConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+	if v == nil {
+		return nil
+	}
+	original := v.(map[string]interface{})
+	if len(original) == 0 {
+		return nil
+	}
+	transformed := make(map[string]interface{})
+	transformed["scenarios"] =
+		flattenDataprocBatchRuntimeConfigAutotuningConfigScenarios(original["scenarios"], d, config)
+	return []interface{}{transformed}
+}
+func flattenDataprocBatchRuntimeConfigAutotuningConfigScenarios(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+	return v
+}
+
+func flattenDataprocBatchRuntimeConfigCohort(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
+	return v
+}
+
 func flattenDataprocBatchEnvironmentConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} {
 	if v == nil {
 		return nil
@@ -1501,6 +1555,20 @@ func expandDataprocBatchRuntimeConfig(v interface{}, d tpgresource.TerraformReso
 		transformed["effective_properties"] = transformedEffectiveProperties
 	}
 
+	transformedAutotuningConfig, err := expandDataprocBatchRuntimeConfigAutotuningConfig(original["autotuning_config"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedAutotuningConfig); val.IsValid() && !tpgresource.IsEmptyValue(val) {
+		transformed["autotuningConfig"] = transformedAutotuningConfig
+	}
+
+	transformedCohort, err := expandDataprocBatchRuntimeConfigCohort(original["cohort"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedCohort); val.IsValid() && !tpgresource.IsEmptyValue(val) {
+		transformed["cohort"] = transformedCohort
+	}
+
 	return transformed, nil
 }
 
@@ -1534,6 +1602,33 @@ func expandDataprocBatchRuntimeConfigEffectiveProperties(v interface{}, d tpgres
 	return m, nil
 }
 
+func expandDataprocBatchRuntimeConfigAutotuningConfig(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+	l := v.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := make(map[string]interface{})
+
+	transformedScenarios, err := expandDataprocBatchRuntimeConfigAutotuningConfigScenarios(original["scenarios"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedScenarios); val.IsValid() && !tpgresource.IsEmptyValue(val) {
+		transformed["scenarios"] = transformedScenarios
+	}
+
+	return transformed, nil
+}
+
+func expandDataprocBatchRuntimeConfigAutotuningConfigScenarios(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+	return v, nil
+}
+
+func expandDataprocBatchRuntimeConfigCohort(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
+	return v, nil
+}
+
 func expandDataprocBatchEnvironmentConfig(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) {
 	l := v.([]interface{})
 	if len(l) == 0 || l[0] == nil {

diff --git a/google/services/dataproc/resource_dataproc_batch_generated_test.go b/google/services/dataproc/resource_dataproc_batch_generated_test.go
@@ -406,6 +406,67 @@ resource "google_dataproc_batch" "example_batch_sparkr" {
 `, context)
 }
 
+func TestAccDataprocBatch_dataprocBatchAutotuningExample(t *testing.T) {
+	t.Parallel()
+
+	context := map[string]interface{}{
+		"project_name":    envvar.GetTestProjectFromEnv(),
+		"prevent_destroy": false,
+		"subnetwork_name": acctest.BootstrapSubnetWithFirewallForDataprocBatches(t, "dataproc-autotuning-test-network", "dataproc-autotuning-test-subnetwork"),
+		"random_suffix":   acctest.RandString(t, 10),
+	}
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckDataprocBatchDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataprocBatch_dataprocBatchAutotuningExample(context),
+			},
+			{
+				ResourceName:            "google_dataproc_batch.example_batch_autotuning",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"batch_id", "labels", "location", "runtime_config.0.properties", "terraform_labels"},
+			},
+		},
+	})
+}
+
+func testAccDataprocBatch_dataprocBatchAutotuningExample(context map[string]interface{}) string {
+	return acctest.Nprintf(`
+resource "google_dataproc_batch" "example_batch_autotuning" {
+
+    batch_id      = "tf-test-batch%{random_suffix}"
+    location      = "us-central1"
+    labels        = {"batch_test": "terraform"}
+
+    runtime_config {
+      version       = "2.2"
+      properties    = { "spark.dynamicAllocation.enabled": "false", "spark.executor.instances": "2" }
+      cohort        = "tf-dataproc-batch-example"
+      autotuning_config {
+        scenarios = ["SCALING", "MEMORY"]
+      }
+    }
+
+    environment_config {
+      execution_config {
+        subnetwork_uri = "%{subnetwork_name}"
+        ttl            = "3600s"
+      }
+    }
+
+    spark_batch {
+      main_class    = "org.apache.spark.examples.SparkPi"
+      args          = ["10"]
+      jar_file_uris = ["file:///usr/lib/spark/examples/jars/spark-examples.jar"]
+    }
+}
+`, context)
+}
+
 func testAccCheckDataprocBatchDestroyProducer(t *testing.T) func(s *terraform.State) error {
 	return func(s *terraform.State) error {
 		for name, rs := range s.RootModule().Resources {

diff --git a/website/docs/r/dataproc_batch.html.markdown b/website/docs/r/dataproc_batch.html.markdown
@@ -266,6 +266,39 @@ resource "google_dataproc_batch" "example_batch_sparkr" {
     }
 }
 ```
+## Example Usage - Dataproc Batch Autotuning
+
+
+```hcl
+resource "google_dataproc_batch" "example_batch_autotuning" {
+
+    batch_id      = "tf-test-batch%{random_suffix}"
+    location      = "us-central1"
+    labels        = {"batch_test": "terraform"}
+
+    runtime_config {
+      version       = "2.2"
+      properties    = { "spark.dynamicAllocation.enabled": "false", "spark.executor.instances": "2" }
+      cohort        = "tf-dataproc-batch-example"
+      autotuning_config {
+        scenarios = ["SCALING", "MEMORY"]
+      }
+    }
+
+    environment_config {
+      execution_config {
+        subnetwork_uri = "default"
+        ttl            = "3600s"
+      }
+    }
+
+    spark_batch {
+      main_class    = "org.apache.spark.examples.SparkPi"
+      args          = ["10"]
+      jar_file_uris = ["file:///usr/lib/spark/examples/jars/spark-examples.jar"]
+    }
+}
+```
 
 ## Argument Reference
 
@@ -344,6 +377,23 @@ The following arguments are supported:
   (Output)
   A mapping of property names to values, which are used to configure workload execution.
 
+* `autotuning_config` -
+  (Optional)
+  Optional. Autotuning configuration of the workload.
+  Structure is [documented below](#nested_autotuning_config).
+
+* `cohort` -
+  (Optional)
+  Optional. Cohort identifier. Identifies families of the workloads having the same shape, e.g. daily ETL jobs.
+
+
+<a name="nested_autotuning_config"></a>The `autotuning_config` block supports:
+
+* `scenarios` -
+  (Optional)
+  Optional. Scenarios for which tunings are applied.
+  Each value may be one of: `SCALING`, `BROADCAST_HASH_JOIN`, `MEMORY`.
+
 <a name="nested_environment_config"></a>The `environment_config` block supports:
 
 * `execution_config` -