diff --git a/.changelog/15511.txt b/.changelog/15511.txt new file mode 100644 index 00000000000..75528c9c472 --- /dev/null +++ b/.changelog/15511.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +container: added `writable_cgroups` field to node `containerd_config` +``` \ No newline at end of file diff --git a/google/services/container/node_config.go b/google/services/container/node_config.go index e0fe90cf2c8..c6ee7b779e4 100644 --- a/google/services/container/node_config.go +++ b/google/services/container/node_config.go @@ -91,6 +91,21 @@ func schemaContainerdConfig() *schema.Schema { }, }}, }, + "writable_cgroups": { + Type: schema.TypeList, + Description: `Parameters for writable cgroups configuration.`, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled": { + Type: schema.TypeBool, + Required: true, + Description: `Whether writable cgroups are enabled.`, + }, + }, + }, + }, }}, } } @@ -1908,6 +1923,7 @@ func expandContainerdConfig(v interface{}) *container.ContainerdConfig { cc := &container.ContainerdConfig{} cc.PrivateRegistryAccessConfig = expandPrivateRegistryAccessConfig(cfg["private_registry_access_config"]) + cc.WritableCgroups = expandWritableCgroups(cfg["writable_cgroups"]) return cc } @@ -1979,6 +1995,26 @@ func expandGCPSecretManagerCertificateConfig(v interface{}) *container.GCPSecret return gcpSMConfig } +func expandWritableCgroups(v interface{}) *container.WritableCgroups { + if v == nil { + return nil + } + ls := v.([]interface{}) + if len(ls) == 0 { + return nil + } + if ls[0] == nil { + return &container.WritableCgroups{} + } + cfg := ls[0].(map[string]interface{}) + + wcg := &container.WritableCgroups{} + if enabled, ok := cfg["enabled"]; ok { + wcg.Enabled = enabled.(bool) + } + return wcg +} + func expandSoleTenantConfig(v interface{}) *container.SoleTenantConfig { if v == nil { return nil @@ -2485,6 +2521,9 @@ func flattenContainerdConfig(c *container.ContainerdConfig) []map[string]interfa if c.PrivateRegistryAccessConfig != nil { r["private_registry_access_config"] = flattenPrivateRegistryAccessConfig(c.PrivateRegistryAccessConfig) } + if c.WritableCgroups != nil { + r["writable_cgroups"] = flattenWritableCgroups(c.WritableCgroups) + } return append(result, r) } @@ -2544,6 +2583,17 @@ func flattenGCPSecretManagerCertificateConfig(c *container.GCPSecretManagerCerti return append(result, r) } +func flattenWritableCgroups(c *container.WritableCgroups) []map[string]interface{} { + result := []map[string]interface{}{} + if c == nil { + return result + } + r := map[string]interface{}{ + "enabled": c.Enabled, + } + return append(result, r) +} + func flattenConfidentialNodes(c *container.ConfidentialNodes) []map[string]interface{} { result := []map[string]interface{}{} if c != nil { diff --git a/google/services/container/resource_container_cluster_meta.yaml b/google/services/container/resource_container_cluster_meta.yaml index b001bd2ba06..392a04030d9 100644 --- a/google/services/container/resource_container_cluster_meta.yaml +++ b/google/services/container/resource_container_cluster_meta.yaml @@ -233,6 +233,7 @@ fields: - api_field: 'nodeConfig.containerdConfig.privateRegistryAccessConfig.certificateAuthorityDomainConfig.fqdns' - api_field: 'nodeConfig.containerdConfig.privateRegistryAccessConfig.certificateAuthorityDomainConfig.gcpSecretManagerCertificateConfig.secretUri' - api_field: 'nodeConfig.containerdConfig.privateRegistryAccessConfig.enabled' + - api_field: 'nodeConfig.containerdConfig.writableCgroups.enabled' - api_field: 'nodeConfig.diskSizeGb' - api_field: 'nodeConfig.diskType' - field: 'node_config.effective_taints.effect' @@ -655,6 +656,7 @@ fields: - api_field: 'nodePoolDefaults.nodeConfigDefaults.containerdConfig.privateRegistryAccessConfig.certificateAuthorityDomainConfig.fqdns' - api_field: 'nodePoolDefaults.nodeConfigDefaults.containerdConfig.privateRegistryAccessConfig.certificateAuthorityDomainConfig.gcpSecretManagerCertificateConfig.secretUri' - api_field: 'nodePoolDefaults.nodeConfigDefaults.containerdConfig.privateRegistryAccessConfig.enabled' + - api_field: 'nodePoolDefaults.nodeConfigDefaults.containerdConfig.writableCgroups.enabled' - api_field: 'nodePoolDefaults.nodeConfigDefaults.gcfsConfig.enabled' - field: 'node_pool_defaults.node_config_defaults.insecure_kubelet_readonly_port_enabled' api_field: 'nodePoolDefaults.nodeConfigDefaults.nodeKubeletConfig.insecureKubeletReadonlyPortEnabled' diff --git a/google/services/container/resource_container_cluster_test.go b/google/services/container/resource_container_cluster_test.go index 68ac6ae5858..4182914b07d 100644 --- a/google/services/container/resource_container_cluster_test.go +++ b/google/services/container/resource_container_cluster_test.go @@ -12932,6 +12932,208 @@ resource "google_container_cluster" "primary" { `, secretID, clusterName, customDomain, networkName, subnetworkName) } +func TestAccContainerCluster_writableCgroups(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + nodePoolName := fmt.Sprintf("tf-test-nodepool-%s", acctest.RandString(t, 10)) + networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster") + subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + // Test enabling writable_cgroups for new node pools via node_pool_defaults. + { + Config: testAccContainerCluster_writableCgroupsEnabled(clusterName, networkName, subnetworkName), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr( + "google_container_cluster.primary", + "node_pool_defaults.0.node_config_defaults.0.containerd_config.0.writable_cgroups.0.enabled", + "true", + ), + ), + }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + // Test disabling writable_cgroups for new node pools via node_pool_defaults. + { + Config: testAccContainerCluster_writableCgroupsDisabled(clusterName, networkName, subnetworkName), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + acctest.ExpectNoDelete(), + }, + }, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr( + "google_container_cluster.primary", + "node_pool_defaults.0.node_config_defaults.0.containerd_config.0.writable_cgroups.0.enabled", + "false", + ), + ), + }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + // Test configuring writable_cgroups on the cluster's default node pool directly via node_config. + { + Config: testAccContainerCluster_withNodeConfigWritableCgroups(clusterName, networkName, subnetworkName), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + acctest.ExpectNoDelete(), + }, + }, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr( + "google_container_cluster.primary", + "node_config.0.containerd_config.0.writable_cgroups.0.enabled", + "true", + ), + ), + }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + // Test configuring writable_cgroups on a named node pool defined within the cluster. + // This change from a default to a named node pool is expected to force recreation. + { + Config: testAccContainerCluster_withNodePoolWritableCgroups(clusterName, nodePoolName, networkName, subnetworkName), + }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + }, + }) +} + +func testAccContainerCluster_writableCgroupsEnabled(clusterName, networkName, subnetworkName string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"] + network = "%s" + subnetwork = "%s" + deletion_protection = false + + node_pool_defaults { + node_config_defaults { + containerd_config { + writable_cgroups { + enabled = true + } + } + } + } +} +`, clusterName, networkName, subnetworkName) +} + +func testAccContainerCluster_writableCgroupsDisabled(clusterName, networkName, subnetworkName string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"] + network = "%s" + subnetwork = "%s" + deletion_protection = false + + node_pool_defaults { + node_config_defaults { + containerd_config { + writable_cgroups { + enabled = false + } + } + } + } +} +`, clusterName, networkName, subnetworkName) +} + +func testAccContainerCluster_withNodePoolWritableCgroups(clusterName, nodePoolName, networkName, subnetworkName string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"] + network = "%s" + subnetwork = "%s" + deletion_protection = false + + node_pool { + name = "%s" + initial_node_count = 1 + node_config { + containerd_config { + writable_cgroups { + enabled = true + } + } + } + } + +} +`, clusterName, networkName, subnetworkName, nodePoolName) +} + +func testAccContainerCluster_withNodeConfigWritableCgroups(clusterName, networkName, subnetworkName string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"] + network = "%s" + subnetwork = "%s" + deletion_protection = false + + node_config { + containerd_config { + writable_cgroups { + enabled = true + } + } + } + +} +`, clusterName, networkName, subnetworkName) +} + func TestAccContainerCluster_withProviderDefaultLabels(t *testing.T) { // The test failed if VCR testing is enabled, because the cached provider config is used. // With the cached provider config, any changes in the provider default labels will not be applied. diff --git a/google/services/container/resource_container_node_pool_test.go b/google/services/container/resource_container_node_pool_test.go index 72f881a027f..c72bcb2e22f 100644 --- a/google/services/container/resource_container_node_pool_test.go +++ b/google/services/container/resource_container_node_pool_test.go @@ -5610,6 +5610,118 @@ resource "google_container_node_pool" "np" { `, secretID, cluster, network, subnetwork, nodepool) } +func TestAccContainerNodePool_writableCgroups(t *testing.T) { + t.Parallel() + + cluster := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + nodepool := fmt.Sprintf("tf-test-nodepool-%s", acctest.RandString(t, 10)) + networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster") + subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerNodePoolDestroyProducer(t), + Steps: []resource.TestStep{ + // Test enabling writable_cgroups on a node pool. + { + Config: testAccContainerNodePool_writableCgroupsEnabled(cluster, nodepool, networkName, subnetworkName), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr( + "google_container_node_pool.np", + "node_config.0.containerd_config.0.writable_cgroups.0.enabled", + "true", + ), + ), + }, + // Test disabling writable_cgroups on a node pool. + { + Config: testAccContainerNodePool_writableCgroupsDisabled(cluster, nodepool, networkName, subnetworkName), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + acctest.ExpectNoDelete(), + }, + }, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr( + "google_container_node_pool.np", + "node_config.0.containerd_config.0.writable_cgroups.0.enabled", + "false", + ), + ), + }, + }, + }) +} + +func testAccContainerNodePool_writableCgroupsEnabled(cluster, nodepool, network, subnetwork string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"] + network = "%s" + subnetwork = "%s" + deletion_protection = false +} + +resource "google_container_node_pool" "np" { + name = "%s" + location = "us-central1-a" + cluster = google_container_cluster.primary.name + initial_node_count = 1 + + node_config { + image_type = "COS_CONTAINERD" + containerd_config { + writable_cgroups { + enabled = true + } + } + } +} +`, cluster, network, subnetwork, nodepool) +} + +func testAccContainerNodePool_writableCgroupsDisabled(cluster, nodepool, network, subnetwork string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"] + network = "%s" + subnetwork = "%s" + deletion_protection = false +} + +resource "google_container_node_pool" "np" { + name = "%s" + location = "us-central1-a" + cluster = google_container_cluster.primary.name + initial_node_count = 1 + + node_config { + image_type = "COS_CONTAINERD" + containerd_config { + writable_cgroups { + enabled = false + } + } + } +} +`, cluster, network, subnetwork, nodepool) +} + func TestAccContainerNodePool_defaultDriverInstallation(t *testing.T) { t.Parallel() diff --git a/website/docs/r/container_cluster.html.markdown b/website/docs/r/container_cluster.html.markdown index cb1ae27f27a..dc7ff1a7252 100644 --- a/website/docs/r/container_cluster.html.markdown +++ b/website/docs/r/container_cluster.html.markdown @@ -1664,6 +1664,10 @@ linux_node_config { } ``` +* `writable_cgroups` (Optional) - Configuration for writable cgroups. This allows containers to have a writable `/sys/fs/cgroup` directory, which is required for some workloads to create their own sub-cgroups. The `writable_cgroups` block supports: + + * `enabled` (Required) - Whether writable cgroups are enabled. + The `vertical_pod_autoscaling` block supports: * `enabled` (Required) - Enables vertical pod autoscaling