Add field metastore_config to google_dataproc_cluster (#5146) (#3577)

* added metastoreConfig to dataproc cluster * Update mmv1/third_party/terraform/resources/resource_dataproc_cluster.go.erb Co-authored-by: Cameron Thornton <camthornton@google.com> Co-authored-by: Cameron Thornton <camthornton@google.com> Signed-off-by: Modular Magician <magic-modules@google.com> Co-authored-by: Cameron Thornton <camthornton@google.com>
hashicorp · Aug 27, 2021 · 9b5d0d8 · 9b5d0d8
1 parent 6d7c089
commit 9b5d0d8
Show file tree

Hide file tree

Showing 4 changed files with 161 additions and 0 deletions.
diff --git a/.changelog/5146.txt b/.changelog/5146.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+dataproc: added field `metastore_config` to `google_dataproc_cluster` (beta)
+```
diff --git a/google-beta/resource_dataproc_cluster.go b/google-beta/resource_dataproc_cluster.go
@@ -62,6 +62,7 @@ var (
 		"cluster_config.0.autoscaling_config",
 		"cluster_config.0.lifecycle_config",
 		"cluster_config.0.endpoint_config",
+		"cluster_config.0.metastore_config",
 	}
 )
 
@@ -667,6 +668,23 @@ by Dataproc`,
 								},
 							},
 						},
+						"metastore_config": {
+							Type:         schema.TypeList,
+							Optional:     true,
+							AtLeastOneOf: clusterConfigKeys,
+							MaxItems:     1,
+							Description:  `Specifies a Metastore configuration.`,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"dataproc_metastore_service": {
+										Type:        schema.TypeString,
+										Required:    true,
+										ForceNew:    true,
+										Description: `Resource name of an existing Dataproc Metastore service.`,
+									},
+								},
+							},
+						},
 					},
 				},
 			},
@@ -935,6 +953,10 @@ func expandClusterConfig(d *schema.ResourceData, config *Config) (*dataproc.Clus
 		conf.EndpointConfig = expandEndpointConfig(cfg)
 	}
 
+	if cfg, ok := configOptions(d, "cluster_config.0.metastore_config"); ok {
+		conf.MetastoreConfig = expandMetastoreConfig(cfg)
+	}
+
 	if cfg, ok := configOptions(d, "cluster_config.0.master_config"); ok {
 		log.Println("[INFO] got master_config")
 		conf.MasterConfig = expandInstanceGroupConfig(cfg)
@@ -1136,6 +1158,14 @@ func expandEndpointConfig(cfg map[string]interface{}) *dataproc.EndpointConfig {
 	return conf
 }
 
+func expandMetastoreConfig(cfg map[string]interface{}) *dataproc.MetastoreConfig {
+	conf := &dataproc.MetastoreConfig{}
+	if v, ok := cfg["dataproc_metastore_service"]; ok {
+		conf.DataprocMetastoreService = v.(string)
+	}
+	return conf
+}
+
 func expandInitializationActions(v interface{}) []*dataproc.NodeInitializationAction {
 	actionList := v.([]interface{})
 
@@ -1403,6 +1433,7 @@ func flattenClusterConfig(d *schema.ResourceData, cfg *dataproc.ClusterConfig) (
 		"preemptible_worker_config": flattenPreemptibleInstanceGroupConfig(d, cfg.SecondaryWorkerConfig),
 		"lifecycle_config":          flattenLifecycleConfig(d, cfg.LifecycleConfig),
 		"endpoint_config":           flattenEndpointConfig(d, cfg.EndpointConfig),
+		"metastore_config":          flattenMetastoreConfig(d, cfg.MetastoreConfig),
 	}
 
 	if len(cfg.InitializationActions) > 0 {
@@ -1509,6 +1540,18 @@ func flattenEndpointConfig(d *schema.ResourceData, ec *dataproc.EndpointConfig)
 	return []map[string]interface{}{data}
 }
 
+func flattenMetastoreConfig(d *schema.ResourceData, ec *dataproc.MetastoreConfig) []map[string]interface{} {
+	if ec == nil {
+		return nil
+	}
+
+	data := map[string]interface{}{
+		"dataproc_metastore_service": ec.DataprocMetastoreService,
+	}
+
+	return []map[string]interface{}{data}
+}
+
 func flattenAccelerators(accelerators []*dataproc.AcceleratorConfig) interface{} {
 	acceleratorsTypeSet := schema.NewSet(schema.HashResource(acceleratorsSchema()), []interface{}{})
 	for _, accelerator := range accelerators {

diff --git a/google-beta/resource_dataproc_cluster_test.go b/google-beta/resource_dataproc_cluster_test.go
@@ -782,6 +782,38 @@ func TestAccDataprocCluster_withAutoscalingPolicy(t *testing.T) {
 	})
 }
 
+func TestAccDataprocCluster_withMetastoreConfig(t *testing.T) {
+	t.Parallel()
+
+	pid := getTestProjectFromEnv()
+	msName_basic := fmt.Sprintf("projects/%s/locations/us-central1/services/metastore-srv", pid)
+	msName_update := fmt.Sprintf("projects/%s/locations/us-central1/services/metastore-srv-update", pid)
+
+	var cluster dataproc.Cluster
+	rnd := randString(t, 10)
+	vcrTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckDataprocClusterDestroy(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataprocCluster_withMetastoreConfig(rnd),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_metastore_config", &cluster),
+					resource.TestCheckResourceAttr("google_dataproc_cluster.with_metastore_config", "cluster_config.0.metastore_config.0.dataproc_metastore_service", msName_basic),
+				),
+			},
+			{
+				Config: testAccDataprocCluster_withMetastoreConfig_update(rnd),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_metastore_config", &cluster),
+					resource.TestCheckResourceAttr("google_dataproc_cluster.with_metastore_config", "cluster_config.0.metastore_config.0.dataproc_metastore_service", msName_update),
+				),
+			},
+		},
+	})
+}
+
 func testAccCheckDataprocClusterDestroy(t *testing.T) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		config := googleProviderConfig(t)
@@ -1732,3 +1764,65 @@ resource "google_dataproc_autoscaling_policy" "asp" {
 }
 `, rnd, rnd)
 }
+
+func testAccDataprocCluster_withMetastoreConfig(rnd string) string {
+	return fmt.Sprintf(`
+resource "google_dataproc_cluster" "with_metastore_config" {
+	name                  = "tf-test-%s"
+	region                = "us-central1"
+
+	cluster_config {
+		metastore_config {
+			dataproc_metastore_service = google_dataproc_metastore_service.ms.name
+		}
+	}
+}
+
+resource "google_dataproc_metastore_service" "ms" {
+	service_id = "metastore-srv"
+	location   = "us-central1"
+	port       = 9080
+	tier       = "DEVELOPER"
+
+	maintenance_window {
+		hour_of_day = 2
+		day_of_week = "SUNDAY"
+	}
+
+	hive_metastore_config {
+		version = "3.1.2"
+	}
+}
+`, rnd)
+}
+
+func testAccDataprocCluster_withMetastoreConfig_update(rnd string) string {
+	return fmt.Sprintf(`
+resource "google_dataproc_cluster" "with_metastore_config" {
+	name                  = "tf-test-%s"
+	region                = "us-central1"
+
+	cluster_config {
+		metastore_config {
+			dataproc_metastore_service = google_dataproc_metastore_service.ms.name
+		}
+	}
+}
+
+resource "google_dataproc_metastore_service" "ms" {
+	service_id = "metastore-srv-update"
+	location   = "us-central1"
+	port       = 9080
+	tier       = "DEVELOPER"
+
+	maintenance_window {
+		hour_of_day = 2
+		day_of_week = "SUNDAY"
+	}
+
+	hive_metastore_config {
+		version = "3.1.2"
+	}
+}
+`, rnd)
+}
diff --git a/website/docs/r/dataproc_cluster.html.markdown b/website/docs/r/dataproc_cluster.html.markdown
@@ -163,6 +163,7 @@ The `cluster_config` block supports:
         initialization_action     { ... }
         encryption_config         { ... }
         endpoint_config           { ... }
+        metastore_config          { ... }
     }
 ```
 
@@ -214,6 +215,9 @@ The `cluster_config` block supports:
 
 * `endpoint_config` (Optional, Beta) The config settings for port access on the cluster.
    Structure defined below.
+
+* `metastore_config` (Optional, Beta) The config setting for metastore service with the cluster.
+   Structure defined below.
 - - -
 
 The `cluster_config.gce_cluster_config` block supports:
@@ -650,6 +654,23 @@ cluster_config {
 * `enable_http_port_access` - (Optional) The flag to enable http access to specific ports
   on the cluster from external sources (aka Component Gateway). Defaults to false.
 
+
+The `metastore_config` block (Optional, Computed, Beta) supports:
+
+```hcl
+cluster_config {
+  metastore_config {
+    dataproc_metastore_service = "projects/projectId/locations/region/services/serviceName"
+  }
+}
+```
+
+* `dataproc_metastore_service` - (Required) Resource name of an existing Dataproc Metastore service.
+
+Only resource names including projectid and location (region) are valid. Examples:
+
+`projects/[projectId]/locations/[dataproc_region]/services/[service-name]`
+
 ## Attributes Reference
 
 In addition to the arguments listed above, the following computed attributes are