Add support for overriding VM template disks in fast provisioned VDCs

.changes/v3.12.0/1206-improvements.md

@@ -0,0 +1,2 @@
+* `vcd_vapp_vm` and `vcd_vm` add field `consolidate_disks_on_create` that  helps to change template
+  disk sizes using `override_template_disk` in fast provisioned VDCs [GH-1206]

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.31.0
 	github.com/kr/pretty v0.2.1
-	github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.1
+	github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.3
 )
 
 require (
@@ -56,7 +56,7 @@ require (
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	github.com/zclconf/go-cty v1.14.1 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/exp v0.0.0-20221114191408-850992195362
+	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect

go.sum

@@ -141,8 +141,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.1 h1:Igwf/RIsxqLtmFOogDvWN/o1aigngsZztuYIeHS6Be0=
-github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.1/go.mod h1:VmJkHY7A3fzF3JPBm+UuWTt5GDQQWRUIo/txM7lySXE=
+github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.3 h1:zjgs2KpSp4KT00ap3yq3UMU/whjbq7IV3Hui7bJ3w/U=
+github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.3/go.mod h1:7zG7TXViQ48JpYZIflmdPu3o30xkQSZ4nO+tZdSq31A=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -154,8 +154,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/exp v0.0.0-20221114191408-850992195362 h1:NoHlPRbyl1VFI6FjwHtPQCN7wAMXI6cKcqrmXhOOfBQ=
-golang.org/x/exp v0.0.0-20221114191408-850992195362/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
@@ -208,8 +208,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

scripts/skip-upgrade-tests.txt

@@ -333,3 +333,5 @@ vcd.TestAccVcdSubscribedCatalog-no-local-copy-subscriber.tf v3.10.0 "Test was no
 vcd.TestAccVcdSubscribedCatalog-no-local-copy-subscriber-update.tf v3.10.0 "Test was not stable, 3.11 introduce improvements in PR 1101"
 vcd.TestAccVcdSubscribedCatalog-no-local-copy-subscriber-sync.tf v3.10.0 "Test was not stable, 3.11 introduce improvements in PR 1101"
 vcd.TestAccVcdAnyAccessControlGroupsstep1.tf v3.11.0 "Test was not stable in 3.11, 3.12 introduced improvements in PR 1194"
+vcd.ResourceSchema-vcd_vapp_vm.tf v3.11.0 "New field 'consolidate_disks_on_create'"
+vcd.ResourceSchema-vcd_vm.tf v3.11.0 "New field 'consolidate_disks_on_create'"

vcd/resource_vcd_nsxt_standalone_vm_test.go

@@ -125,7 +125,8 @@ func TestAccVcdNsxtStandaloneVmTemplate(t *testing.T) {
 				// it is reported during import
 				// "network_dhcp_wait_seconds" is a user setting and cannot be imported
 				ImportStateVerifyIgnore: []string{"template_name", "catalog_name",
-					"accept_all_eulas", "power_on", "computer_name", "prevent_update_power_off", "network.1.ip", "network_dhcp_wait_seconds"},
+					"accept_all_eulas", "power_on", "computer_name", "prevent_update_power_off", "network.1.ip",
+					"network_dhcp_wait_seconds", "consolidate_disks_on_create"},
 			},
 			// This step ensures that VM and disk are removed, but networks are left
 			{

vcd/resource_vcd_standalone_vm_test.go

@@ -87,7 +87,8 @@ func TestAccVcdStandaloneVmTemplate(t *testing.T) {
 				ImportStateIdFunc: importStateIdOrgVdcObject(standaloneVmName),
 				// These fields can't be retrieved from user data
 				ImportStateVerifyIgnore: []string{"template_name", "catalog_name",
-					"accept_all_eulas", "power_on", "computer_name", "prevent_update_power_off"},
+					"accept_all_eulas", "power_on", "computer_name", "prevent_update_power_off",
+					"consolidate_disks_on_create"},
 			},
 		},
 	})

vcd/resource_vcd_vapp_vm.go

@@ -381,6 +381,13 @@ func vmSchemaFunc(vmType typeOfVm) map[string]*schema.Schema {
 			Optional: true,
 			Set:      resourceVcdVmIndependentDiskHash,
 		},
+		"consolidate_disks_on_create": {
+			Type:        schema.TypeBool,
+			Optional:    true,
+			Default:     false,
+			ForceNew:    true,
+			Description: "Consolidates disks during creation and allows to change disk size using 'override_template_disk' in fast provisioned VDCs",
+		},
 		"override_template_disk": {
 			Type:        schema.TypeSet,
 			Optional:    true,
@@ -1170,6 +1177,26 @@ func createVmFromTemplate(d *schema.ResourceData, meta interface{}, vmType typeO
 		return nil, fmt.Errorf("error refreshing VM %s : %s", vmName, err)
 	}
 
+	// Check if disk consolidation is requested - it is a required operation in fast provisioned
+	// VDCs when override_template_disk tries to increase disk size, but there can also be other use
+	// cases.
+	// Such fields are processed:
+	// * consolidate_disks_on_create
+	//
+	// Note. Consolidating disks requires "vApp: VM Migrate, Force Undeploy, Relocate, Consolidate"
+	// right
+	if d.Get("consolidate_disks_on_create").(bool) {
+		util.Logger.Printf("[INFO] disk consolidation is requested with field 'consolidate_disks_on_create': %s", err)
+		err := vm.ConsolidateDisks()
+		if err != nil {
+			return nil, fmt.Errorf("error occurred while consolidating disks for VM '%s': %s", vm.VM.Name, err)
+		}
+	}
+
+	if err := vm.Refresh(); err != nil {
+		return nil, fmt.Errorf("error refreshing VM %s : %s", vmName, err)
+	}
+
 	// update existing internal disks in template (it is only applicable to VMs created
 	// Such fields are processed:
 	// * override_template_disk

vcd/resource_vcd_vapp_vm_4types_test.go

@@ -2079,3 +2079,148 @@ func testAccCheckVcdVappPowerState(orgName, vdcName string, vappName string, exp
 		return nil
 	}
 }
+
+// TestAccVcdVAppVm_2typesOverrideDiskFastProvisionedVdc checks that `consolidate_disks_on_create`
+// performs disk consolidation, which in turn allows to use 'override_template_disk' for growing
+// template based VMs (vApp and standalone) at the time of creation in fast provisioned VDCs
+func TestAccVcdVAppVm_2typesOverrideDiskFastProvisionedVdc(t *testing.T) {
+	preTestChecks(t)
+
+	var params = StringMap{
+		"TestName":           t.Name(),
+		"Org":                testConfig.VCD.Org,
+		"Vdc":                testConfig.Nsxt.Vdc,
+		"Catalog":            testConfig.VCD.Catalog.NsxtBackedCatalogName,
+		"CatalogItem":        testConfig.VCD.Catalog.CatalogItemWithMultiVms,
+		"VmNameInTemplate1":  testConfig.VCD.Catalog.VmName1InMultiVmItem,
+		"VmNameInTemplate2":  testConfig.VCD.Catalog.VmName2InMultiVmItem,
+		"Media":              testConfig.Media.NsxtBackedMediaName,
+		"NsxtEdgeGateway":    testConfig.Nsxt.EdgeGateway,
+		"StorageProfileName": testConfig.VCD.NsxtProviderVdc.StorageProfile,
+
+		"Tags": "vapp vm",
+	}
+	testParamsNotEmpty(t, params)
+
+	configTextStep1 := templateFill(testAccVcdVAppVm_4typesOverrideDiskFastProvisionedVdc, params)
+
+	if vcdShortTest {
+		t.Skip(acceptanceTestsSkipped)
+		return
+	}
+	debugPrintf("#[DEBUG] CONFIGURATION: %s\n", configTextStep1)
+	resource.Test(t, resource.TestCase{
+		ProviderFactories: testAccProviders,
+		CheckDestroy: resource.ComposeAggregateTestCheckFunc(
+			testAccCheckVcdNsxtVAppVmDestroy(t.Name()+"-template-vm"),
+			testAccCheckVcdStandaloneVmDestroy(t.Name()+"-template-standalone-vm", testConfig.VCD.Org, testConfig.Nsxt.Vdc),
+		),
+		Steps: []resource.TestStep{
+			{
+				Config: configTextStep1,
+				Check: resource.ComposeAggregateTestCheckFunc(
+					// Template vApp VM checks
+					resource.TestCheckResourceAttr("vcd_vapp_vm.template-vm", "vm_type", "vcd_vapp_vm"),
+					resource.TestCheckResourceAttr("vcd_vapp_vm.template-vm", "name", t.Name()+"-template-vapp-vm"),
+					resource.TestCheckResourceAttr("vcd_vapp_vm.template-vm", "description", t.Name()+"-template-vapp-vm"),
+					resource.TestCheckResourceAttr("vcd_vapp_vm.template-vm", "power_on", "true"),
+					resource.TestCheckResourceAttr("vcd_vapp_vm.template-vm", "consolidate_disks_on_create", "true"),
+					resource.TestCheckResourceAttr("vcd_vapp_vm.template-vm", "status_text", "POWERED_ON"),
+					testAccCheckVcdVMPowerState(testConfig.VCD.Org, testConfig.Nsxt.Vdc, t.Name()+"-template-vm", t.Name()+"-template-vapp-vm", "POWERED_ON"),
+					resource.TestCheckOutput("vcd_vapp_vm_disk_size", "20480"),
+
+					// Standalone template VM checks
+					resource.TestCheckResourceAttr("vcd_vm.template-vm", "vm_type", "vcd_vm"),
+					resource.TestCheckResourceAttr("vcd_vm.template-vm", "name", t.Name()+"-template-standalone-vm"),
+					resource.TestCheckResourceAttr("vcd_vm.template-vm", "description", t.Name()+"-template-standalone-vm"),
+					resource.TestCheckResourceAttr("vcd_vm.template-vm", "power_on", "true"),
+					resource.TestCheckResourceAttr("vcd_vm.template-vm", "consolidate_disks_on_create", "true"),
+					resource.TestCheckResourceAttr("vcd_vm.template-vm", "status_text", "POWERED_ON"),
+					testAccCheckVcdVMPowerState(testConfig.VCD.Org, testConfig.Nsxt.Vdc, "", t.Name()+"-template-standalone-vm", "POWERED_ON"),
+					resource.TestCheckOutput("vcd_vm_disk_size", "20480"),
+				),
+			},
+		},
+	})
+	postTestChecks(t)
+}
+
+const testAccVcdVAppVm_4typesOverrideDiskFastProvisionedVdc = `
+data "vcd_catalog" "{{.Catalog}}" {
+  org  = "{{.Org}}"
+  name = "{{.Catalog}}"
+}
+
+data "vcd_catalog_vapp_template" "multivm" {
+  org         = "{{.Org}}"
+  catalog_id = data.vcd_catalog.{{.Catalog}}.id
+  name       = "{{.CatalogItem}}"
+}
+
+resource "vcd_vapp" "template-vm" {
+  org         = "{{.Org}}"
+  vdc         = "{{.Vdc}}"
+  name        = "{{.TestName}}-template-vm"
+  description = "vApp for Template VM description"
+  power_on    = true
+}
+
+resource "vcd_vapp_vm" "template-vm" {
+  org  = "{{.Org}}"
+  vdc  = "{{.Vdc}}"
+
+  vapp_template_id    = data.vcd_catalog_vapp_template.multivm.id
+  vm_name_in_template = "{{.VmNameInTemplate1}}"
+
+  vapp_name   = vcd_vapp.template-vm.name
+  name        = "{{.TestName}}-template-vapp-vm"
+  description = "{{.TestName}}-template-vapp-vm"
+  power_on    = true
+
+  consolidate_disks_on_create = true
+
+  override_template_disk {
+    bus_type         = "parallel"
+    size_in_mb       = "20480"
+    bus_number       = 0
+    unit_number      = 0
+    iops             = 0
+    storage_profile  = "{{.StorageProfileName}}"
+  }
+
+  prevent_update_power_off = true
+}
+
+resource "vcd_vm" "template-vm" {
+  org  = "{{.Org}}"
+  vdc  = "{{.Vdc}}"
+
+  vapp_template_id    = data.vcd_catalog_vapp_template.multivm.id
+  vm_name_in_template = "{{.VmNameInTemplate1}}"
+
+  name        = "{{.TestName}}-template-standalone-vm"
+  description = "{{.TestName}}-template-standalone-vm"
+  power_on    = true
+
+  consolidate_disks_on_create = true
+
+  override_template_disk {
+    bus_type         = "parallel"
+    size_in_mb       = "20480"
+    bus_number       = 0
+    unit_number      = 0
+    iops             = 0
+    storage_profile  = "{{.StorageProfileName}}"
+  }
+
+  prevent_update_power_off = true
+}
+
+output "vcd_vapp_vm_disk_size" {
+  value = tolist(vcd_vapp_vm.template-vm.override_template_disk)[0].size_in_mb
+}
+
+output "vcd_vm_disk_size" {
+	value = tolist(vcd_vm.template-vm.override_template_disk)[0].size_in_mb
+}
+`

vcd/resource_vcd_vapp_vm_test.go

@@ -86,7 +86,8 @@ func TestAccVcdVAppVm_Basic(t *testing.T) {
 				ImportStateIdFunc: importStateIdVappObject(vappName2, vmName, testConfig.VCD.Vdc),
 				// These fields can't be retrieved from user data
 				ImportStateVerifyIgnore: []string{"template_name", "catalog_name",
-					"accept_all_eulas", "power_on", "computer_name", "prevent_update_power_off"},
+					"accept_all_eulas", "power_on", "computer_name", "prevent_update_power_off",
+					"consolidate_disks_on_create"},
 			},
 		},
 	})

vcd/resource_vcd_vm_internal_disk_test.go

@@ -150,7 +150,8 @@ func TestAccVcdVmInternalDisk(t *testing.T) {
 				ImportStateVerify: true,
 				ImportStateIdFunc: importStateIdVmObject(testConfig.VCD.Org, vdcName, vappName, vmName, "3000"),
 				// These fields can't be retrieved
-				ImportStateVerifyIgnore: []string{"org", "vdc", "allow_vm_reboot", "thin_provisioned"},
+				ImportStateVerifyIgnore: []string{"org", "vdc", "allow_vm_reboot", "thin_provisioned",
+					"consolidate_disks_on_create"},
 			},
 		},
 	})

website/docs/r/vapp_vm.html.markdown

@@ -197,6 +197,10 @@ resource "vcd_vapp_vm" "internalDiskOverride" {
   cpus             = 2
   cpu_cores        = 1
 
+  # Fast provisioned VDCs require disks to be consolidated
+  # if their size is to be changed
+  # consolidate_disks_on_create = true 
+
   override_template_disk {
     bus_type        = "paravirtual"
     size_in_mb      = "22384"
@@ -479,6 +483,11 @@ example for usage details.
 * `description`  - (Optional; *v2.9+*) The VM description. Note: for VM from Template `description` is read only. Currently, this field has
   the description of the OVA used to create the VM.
 * `override_template_disk` - (Optional; *v2.7+*) Allows to update internal disk in template before first VM boot. Disk is matched by `bus_type`, `bus_number` and `unit_number`. See [Override template Disk](#override-template-disk) below for details.
+* `consolidate_disks_on_create` - (Optional; *3.12+*) Performs disk consolidation during creation.
+  The main use case is when one wants to grow template disk size using `override_template_disk` in
+  fast provisioned VDCs. **Note:** Consolidating disks requires right `vApp: VM Migrate, Force
+  Undeploy, Relocate, Consolidate`. This operation _may take long time_ depending on disk size and
+  storage performance.
 * `network_dhcp_wait_seconds` - (Optional; *v2.7+*) Optional number of seconds to try and wait for DHCP IP (only valid
   for adapters in `network` block with `ip_allocation_mode=DHCP`). It constantly checks if IP is present so the time given
   is a maximum. VM must be powered on and _at least one_ of the following _must be true_:
@@ -584,7 +593,8 @@ example for usage details.
 Allows to update internal disk in template before first VM boot. Disk is matched by `bus_type`, `bus_number` and `unit_number`.
 Changes are ignored on update. This part isn't reread on refresh. To manage internal disk later please use [`vcd_vm_internal_disk`](/providers/vmware/vcd/latest/docs/resources/vm_internal_disk) resource.
 
-~> **Note:** Managing disks in VM is possible only when VDC fast provisioned is disabled.
+~> **Note:** Managing disks in VM with fast provisioned VDC require
+[`consolidate_disks_on_create`](#consolidate_disks_on_create) option.
 
 * `bus_type` - (Required) The type of disk controller. Possible values: `ide`, `parallel`( LSI Logic Parallel SCSI),
   `sas`(LSI Logic SAS (SCSI)), `paravirtual`(Paravirtual (SCSI)), `sata`, `nvme`. **Note** `nvme` requires *v3.5.0+* and