Upgrade TF azurerm v2 and NatGateway pubIP migration

charts/images.yaml

@@ -2,7 +2,7 @@ images:
 - name: terraformer
   sourceRepository: github.com/gardener/terraformer
   repository: eu.gcr.io/gardener-project/gardener/terraformer
-  tag: "v1.4.0"
+  tag: "v1.5.0"
 
 - name: cloud-controller-manager
   sourceRepository: github.com/kubernetes/kubernetes

charts/internal/azure-infra/templates/main.tf

@@ -3,6 +3,8 @@ provider "azurerm" {
   tenant_id       = "{{ required "azure.tenantID is required" .Values.azure.tenantID }}"
   client_id       = var.CLIENT_ID
   client_secret   = var.CLIENT_SECRET
+
+  features {}
 }
 
 {{ if .Values.create.resourceGroup -}}
@@ -47,10 +49,8 @@ resource "azurerm_subnet" "workers" {
   virtual_network_name      = data.azurerm_virtual_network.vnet.name
   resource_group_name       = data.azurerm_virtual_network.vnet.resource_group_name
   {{- end }}
-  address_prefix            = "{{ required "networks.worker is required" .Values.networks.worker }}"
+  address_prefixes          = ["{{ required "networks.worker is required" .Values.networks.worker }}"]
   service_endpoints         = [{{range $index, $serviceEndpoint := .Values.resourceGroup.subnet.serviceEndpoints}}{{if $index}},{{end}}"{{$serviceEndpoint}}"{{end}}]
-  route_table_id            = azurerm_route_table.workers.id
-  network_security_group_id = azurerm_network_security_group.workers.id
 }
 
 resource "azurerm_route_table" "workers" {
@@ -109,14 +109,23 @@ resource "azurerm_nat_gateway" "nat" {
   resource_group_name     = data.azurerm_resource_group.rg.name
   {{- end }}
   sku_name                = "Standard"
-  public_ip_address_ids   = [azurerm_public_ip.natip.id]
-  {{- if .Values.natGateway }}
-  {{- if .Values.natGateway.idleConnectionTimeoutMinutes }}
+  {{ if .Values.natGateway -}}
+  {{ if .Values.natGateway.idleConnectionTimeoutMinutes -}}
   idle_timeout_in_minutes = {{ .Values.natGateway.idleConnectionTimeoutMinutes }}
   {{- end }}
+
+  # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+  {{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}}
+  public_ip_address_ids   = []
+  {{- end }}
   {{- end }}
 }
 
+resource "azurerm_nat_gateway_public_ip_association" "natip-association" {
+  nat_gateway_id       = azurerm_nat_gateway.nat.id
+  public_ip_address_id = azurerm_public_ip.natip.id
+}
+
 resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" {
   subnet_id      = azurerm_subnet.workers.id
   nat_gateway_id = azurerm_nat_gateway.nat.id
@@ -216,4 +225,4 @@ output "{{ .Values.outputKeys.identityID }}" {
 output "{{ .Values.outputKeys.identityClientID }}" {
   value = data.azurerm_user_assigned_identity.identity.client_id
 }
-{{- end }}
+{{- end }}

charts/internal/azure-infra/values.yaml

@@ -15,6 +15,11 @@ create:
   # name: identity-name
   # resourceGroup: identity-resource-group
 
+natGateway:
+  idleConnectionTimeoutMinutes:
+  # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+  migrateNatGatewayToIPAssociation: false
+
 resourceGroup:
   name: my-resource-group
   vnet:
@@ -42,7 +47,3 @@ outputKeys:
   securityGroupName: securityGroupName
   # identityID: managedIdentityID
   # identityClientID: managedIdentityClientID
-
-natGateway:
-  idleConnectionTimeoutMinutes:
-

hack/api-reference/api.md

@@ -645,6 +645,19 @@ bool
 <p>Zoned indicates whether the cluster uses zones</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>natGatewayPublicIpMigrated</code></br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
+TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.MachineImage">MachineImage

pkg/apis/azure/types_infrastructure.go

@@ -70,6 +70,9 @@ type InfrastructureStatus struct {
 	Identity *IdentityStatus
 	// Zoned indicates whether the cluster uses zones
 	Zoned bool
+	// NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+	NatGatewayPublicIPMigrated bool
 }
 
 // NetworkStatus is the current status of the infrastructure networks.

pkg/apis/azure/v1alpha1/types_infrastructure.go

@@ -78,6 +78,10 @@ type InfrastructureStatus struct {
 	// Zoned indicates whether the cluster uses zones
 	// +optional
 	Zoned bool `json:"zoned,omitempty"`
+	// NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+	// +optional
+	NatGatewayPublicIPMigrated bool `json:"natGatewayPublicIpMigrated,omitempty"`
 }
 
 // NetworkStatus is the current status of the infrastructure networks.

pkg/controller/infrastructure/actuator.go

@@ -45,12 +45,7 @@ func NewActuator() infrastructure.Actuator {
 	}
 }
 
-func (a *actuator) updateProviderStatus(
-	ctx context.Context,
-	tf terraformer.Terraformer,
-	infra *extensionsv1alpha1.Infrastructure,
-	config *api.InfrastructureConfig,
-) error {
+func (a *actuator) updateProviderStatus(ctx context.Context, tf terraformer.Terraformer, infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) error {
 	status, err := infrainternal.ComputeStatus(tf, config)
 	if err != nil {
 		return err

pkg/internal/infrastructure/terraform.go

@@ -140,6 +140,14 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli
 		}
 	}
 
+	// Checks if the Gardener managed NatGateway public ip needs to be migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+	natGatewayIPMigrationRequired, err := isNatGatewayIPMigrationRequired(infra, config)
+	if err != nil {
+		return nil, err
+	}
+	natGatewayConfig["migrateNatGatewayToIPAssociation"] = natGatewayIPMigrationRequired
+
 	if config.Identity != nil && config.Identity.Name != "" && config.Identity.ResourceGroup != "" {
 		identityConfig = map[string]interface{}{
 			"name":          config.Identity.Name,
@@ -227,6 +235,9 @@ type TerraformState struct {
 	IdentityID string
 	// IdentityClientID is the client id of the identity.
 	IdentityClientID string
+	// NatGatewayIPMigrated is the indicator if the nat gateway ip is migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+	NatGatewayIPMigrated string
 }
 
 // ExtractTerraformState extracts the TerraformState from the given Terraformer.
@@ -289,6 +300,10 @@ func ExtractTerraformState(tf terraformer.Terraformer, config *api.Infrastructur
 		tfState.IdentityClientID = vars[TerraformerOutputKeyIdentityClientID]
 	}
 
+	if config.Networks.NatGateway != nil && config.Networks.NatGateway.Enabled {
+		tfState.NatGatewayIPMigrated = "true"
+	}
+
 	return &tfState, nil
 }
 
@@ -344,6 +359,11 @@ func StatusFromTerraformState(state *TerraformState) *apiv1alpha1.Infrastructure
 		})
 	}
 
+	// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+	if state.NatGatewayIPMigrated == "true" {
+		tfState.NatGatewayPublicIPMigrated = true
+	}
+
 	return &tfState
 }
 
@@ -418,3 +438,25 @@ func findDomainCounts(cluster *controller.Cluster, infra *extensionsv1alpha1.Inf
 		updateDomains: *updateDomainCount,
 	}, nil
 }
+
+// isNatGatewayIPMigrationRequired checks if the Gardener managed NatGateway public ip needs to be migrated.
+// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+func isNatGatewayIPMigrationRequired(infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (bool, error) {
+	if config.Networks.NatGateway == nil || !config.Networks.NatGateway.Enabled {
+		return false, nil
+	}
+
+	if infra.Status.ProviderStatus == nil {
+		return false, nil
+	}
+
+	infrastructureStatus, err := helper.InfrastructureStatusFromInfrastructure(infra)
+	if err != nil {
+		return false, err
+	}
+
+	if infrastructureStatus.NatGatewayPublicIPMigrated {
+		return false, nil
+	}
+	return true, nil
+}

pkg/internal/infrastructure/terraform_test.go

@@ -21,7 +21,6 @@ import (
 	apiv1alpha1 "github.com/gardener/gardener-extension-provider-azure/pkg/apis/azure/v1alpha1"
 	"github.com/gardener/gardener-extension-provider-azure/pkg/internal"
 	"github.com/gardener/gardener/extensions/pkg/controller"
-
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	. "github.com/onsi/ginkgo"
@@ -178,7 +177,9 @@ var _ = Describe("Terraform", func() {
 				"securityGroupName": TerraformerOutputKeySecurityGroupName,
 			}
 
-			expectedNatGatewayValues = map[string]interface{}{}
+			expectedNatGatewayValues = map[string]interface{}{
+				"migrateNatGatewayToIPAssociation": false,
+			}
 
 			expectedValues = map[string]interface{}{
 				"azure":         expectedAzureValues,
@@ -330,6 +331,50 @@ var _ = Describe("Terraform", func() {
 				Expect(err).To(Not(HaveOccurred()))
 				Expect(values).To(BeEquivalentTo(expectedValues))
 			})
+
+			// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+			Context("NatGateway Gardener managed IP migration", func() {
+				BeforeEach(func() {
+					config.Networks.NatGateway = &api.NatGatewayConfig{
+						Enabled: true,
+					}
+					expectedCreateValues["natGateway"] = true
+				})
+
+				It("should migrate the NatGateway IP as it is not yet migrated", func() {
+					infrastructureStatus := api.InfrastructureStatus{
+						NatGatewayPublicIPMigrated: false,
+					}
+					infrastructureStatusMarshalled, err := json.Marshal(infrastructureStatus)
+					Expect(err).NotTo(HaveOccurred())
+
+					infra.Status.ProviderStatus = &runtime.RawExtension{
+						Raw: infrastructureStatusMarshalled,
+					}
+
+					expectedNatGatewayValues["migrateNatGatewayToIPAssociation"] = true
+					values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster)
+					Expect(err).To(Not(HaveOccurred()))
+					Expect(values).To(BeEquivalentTo(expectedValues))
+				})
+
+				It("should not migrate the NatGateway IP as it is already migrated", func() {
+					infrastructureStatus := api.InfrastructureStatus{
+						NatGatewayPublicIPMigrated: true,
+					}
+					infrastructureStatusMarshalled, err := json.Marshal(infrastructureStatus)
+					Expect(err).NotTo(HaveOccurred())
+
+					infra.Status.ProviderStatus = &runtime.RawExtension{
+						Raw: infrastructureStatusMarshalled,
+					}
+
+					expectedNatGatewayValues["migrateNatGatewayToIPAssociation"] = false
+					values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster)
+					Expect(err).To(Not(HaveOccurred()))
+					Expect(values).To(BeEquivalentTo(expectedValues))
+				})
+			})
 		})
 	})
 

pkg/internal/terraform.go

@@ -56,11 +56,7 @@ func TerraformerEnvVars(secretRef corev1.SecretReference) []corev1.EnvVar {
 }
 
 // NewTerraformer initializes a new Terraformer.
-func NewTerraformer(
-	restConfig *rest.Config,
-	purpose string,
-	infra *extensionsv1alpha1.Infrastructure,
-) (terraformer.Terraformer, error) {
+func NewTerraformer(restConfig *rest.Config, purpose string, infra *extensionsv1alpha1.Infrastructure) (terraformer.Terraformer, error) {
 	tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, infra.Namespace, infra.Name, imagevector.TerraformerImage())
 	if err != nil {
 		return nil, err
@@ -73,11 +69,7 @@ func NewTerraformer(
 }
 
 // NewTerraformerWithAuth initializes a new Terraformer that has the azure auth credentials.
-func NewTerraformerWithAuth(
-	restConfig *rest.Config,
-	purpose string,
-	infra *extensionsv1alpha1.Infrastructure,
-) (terraformer.Terraformer, error) {
+func NewTerraformerWithAuth(restConfig *rest.Config, purpose string, infra *extensionsv1alpha1.Infrastructure) (terraformer.Terraformer, error) {
 	tf, err := NewTerraformer(restConfig, purpose, infra)
 	if err != nil {
 		return nil, err