Add support for cleaning Azure resources

Signed-off-by: Kyle Squizzato <ksquizzato@mirantis.com>
Mirantis · Oct 3, 2024 · 672d5f2 · 672d5f2
1 parent 7ffbf66
commit 672d5f2
Show file tree

Hide file tree

Showing 10 changed files with 64 additions and 15 deletions.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -124,6 +124,7 @@ jobs:
  AWS_REGION: us-west-2
  AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}
+ AZURE_REGION: westus2
  AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }}
  AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
  AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
@@ -233,6 +234,9 @@ jobs:
  AWS_REGION: us-west-2
  AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}
+ AZURE_REGION: westus2
+ AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
  CLUSTER_NAME: '${{ needs.build.outputs.clustername }}'
  run: |
  make dev-aws-nuke
+ make dev-azure-nuke
diff --git a/.gitignore b/.gitignore
@@ -15,7 +15,7 @@ go.work
 go.work.sum
 
 # cloud-nuke config
-*cloud_nuke.yaml
+*cloud-nuke.yaml
 
 # Test artifacts
 test/e2e/*.log

diff --git a/Makefile b/Makefile
@@ -205,8 +205,6 @@ REGISTRY_PORT ?= 5001
 REGISTRY_REPO ?= oci://127.0.0.1:$(REGISTRY_PORT)/charts
 DEV_PROVIDER ?= aws
 REGISTRY_IS_OCI = $(shell echo $(REGISTRY_REPO) | grep -q oci && echo true || echo false)
-CLUSTER_NAME ?= $(shell $(YQ) '.metadata.name' ./config/dev/deployment.yaml)
-
 AWS_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS}
 
 ifndef ignore-not-found
@@ -341,13 +339,23 @@ dev-mcluster-delete: envsubst
 dev-creds-apply: dev-$(DEV_PROVIDER)-creds
 
 .PHONY: dev-aws-nuke
-dev-aws-nuke: envsubst awscli yq cloud-nuke ## Warning: Destructive! Nuke all AWS resources deployed by 'DEV_PROVIDER=aws dev-provider-apply', prefix with CLUSTER_NAME to nuke a specific cluster.
+dev-aws-nuke: envsubst awscli yq cloud-nuke ## Warning: Destructive! Nuke all AWS resources deployed by 'DEV_PROVIDER=aws dev-mcluster-apply'
  @CLUSTER_NAME=$(CLUSTER_NAME) YQ=$(YQ) AWSCLI=$(AWSCLI) bash -c "./scripts/aws-nuke-ccm.sh elb"
- @CLUSTER_NAME=$(CLUSTER_NAME) $(ENVSUBST) < config/dev/cloud_nuke.yaml.tpl > config/dev/cloud_nuke.yaml
- DISABLE_TELEMETRY=true $(CLOUDNUKE) aws --region $$AWS_REGION --force --config config/dev/cloud_nuke.yaml --resource-type vpc,eip,nat-gateway,ec2,ec2-subnet,elb,elbv2,ebs,internet-gateway,network-interface,security-group
- @rm config/dev/cloud_nuke.yaml
+ @CLUSTER_NAME=$(CLUSTER_NAME) $(ENVSUBST) < config/dev/aws-cloud-nuke.yaml.tpl > config/dev/aws-cloud-nuke.yaml
+ DISABLE_TELEMETRY=true $(CLOUDNUKE) aws --region $$AWS_REGION --force --config config/dev/aws-cloud-nuke.yaml --resource-type vpc,eip,nat-gateway,ec2,ec2-subnet,elb,elbv2,ebs,internet-gateway,network-interface,security-group
+ @rm config/dev/aws-cloud-nuke.yaml
  @CLUSTER_NAME=$(CLUSTER_NAME) YQ=$(YQ) AWSCLI=$(AWSCLI) bash -c "./scripts/aws-nuke-ccm.sh ebs"
 
+.PHONY: dev-azure-nuke
+dev-azure-nuke: envsubst azure-nuke ## Warning: Destructive! Nuke all Azure resources deployed by 'DEV_PROVIDER=azure dev-mcluster-apply'
+ @if [ "$(CLUSTER_NAME)" == "" ] || [ "$(AZURE_TENANT_ID)" == "" ] || [ "$(AZURE_REGION)" == "" ]; then \
+ echo "CLUSTER_NAME, AZURE_TENANT_ID and AZURE_REGION must be set"; \
+ exit 1; \
+ fi
+ @CLUSTER_NAME=$(CLUSTER_NAME) $(ENVSUBST) < config/dev/azure-cloud-nuke.yaml.tpl > config/dev/azure-cloud-nuke.yaml
+ $(AZURENUKE) run --config config/dev/azure-cloud-nuke.yaml --force --no-dry-run
+ @rm config/dev/azure-cloud-nuke.yaml
+
 .PHONY: cli-install
 cli-install: clusterawsadm clusterctl cloud-nuke envsubst yq awscli ## Install the necessary CLI tools for deployment, development and testing.
 
@@ -379,6 +387,7 @@ YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION)
 CLUSTERAWSADM ?= $(LOCALBIN)/clusterawsadm
 CLUSTERCTL ?= $(LOCALBIN)/clusterctl
 CLOUDNUKE ?= $(LOCALBIN)/cloud-nuke
+AZURENUKE ?= $(LOCALBIN)/azure-nuke
 ADDLICENSE ?= $(LOCALBIN)/addlicense-$(ADDLICENSE_VERSION)
 ENVSUBST ?= $(LOCALBIN)/envsubst-$(ENVSUBST_VERSION)
 AWSCLI ?= $(LOCALBIN)/aws
@@ -391,6 +400,7 @@ HELM_VERSION ?= v3.15.1
 KIND_VERSION ?= v0.23.0
 YQ_VERSION ?= v4.44.2
 CLOUDNUKE_VERSION = v0.37.1
+AZURENUKE_VERSION = v1.1.0
 CLUSTERAWSADM_VERSION ?= v2.5.2
 CLUSTERCTL_VERSION ?= v1.7.3
 ADDLICENSE_VERSION ?= v1.1.1
@@ -450,6 +460,12 @@ $(CLOUDNUKE): | $(LOCALBIN)
  curl -sL https://github.com/gruntwork-io/cloud-nuke/releases/download/$(CLOUDNUKE_VERSION)/cloud-nuke_$(OS)_$(ARCH) -o $(CLOUDNUKE)
  chmod +x $(CLOUDNUKE)
 
+.PHONY: azure-nuke
+azure-nuke: $(AZURENUKE) ## Download azure-nuke locally if necessary.
+$(AZURENUKE): | $(LOCALBIN)
+ curl -sL https://github.com/ekristen/azure-nuke/releases/download/$(AZURENUKE_VERSION)/azure-nuke-$(AZURENUKE_VERSION)-$(OS)-$(ARCH).tar.gz -o /tmp/azure-nuke.tar.gz
+ tar xvf /tmp/azure-nuke.tar.gz -C $(LOCALBIN) azure-nuke
+
 .PHONY: clusterawsadm
 clusterawsadm: $(CLUSTERAWSADM) ## Download clusterawsadm locally if necessary.
 $(CLUSTERAWSADM): | $(LOCALBIN)

diff --git a/config/dev/cloud_nuke.yaml.tpl → config/dev/aws-cloud-nuke.yaml.tpl b/config/dev/cloud_nuke.yaml.tpl → config/dev/aws-cloud-nuke.yaml.tpl
@@ -1,14 +1,13 @@
 # This config file is used by cloud-nuke to clean up named resources associated
 # with a specific managed cluster across an AWS account. CLUSTER_NAME is
-# typically the metadata.name of the Deployment.
+# typically the metadata.name of the ManagedCluster.
 # The resources listed here are ALL of the potential resources that can be
 # filtered by cloud-nuke, except for IAM resources since we'll never touch those.
 # See: https://github.com/gruntwork-io/cloud-nuke?tab=readme-ov-file#whats-supported
 #
 # Usage:
-# - 'make aws-dev-nuke' will nuke resources affiliated with config/dev/deployment.yaml
-# - 'CLUSTER_NAME=foo make aws-dev-nuke' will nuke resources affiliated with an AWS cluster named 'foo'
-# Check cluster names with 'kubectl get clusters -n hmc-system'
+# - 'CLUSTER_NAME=foo make dev-aws-nuke' will nuke resources affiliated with an AWS cluster named 'foo'
+# Check cluster names with 'kubectl get managedcluster.hmc.mirantis.com -n hmc-system'
 
 ACM:
  include:

diff --git a/config/dev/azure-cloud-nuke.yaml.tpl b/config/dev/azure-cloud-nuke.yaml.tpl
@@ -0,0 +1,26 @@
+# This config file is used by azure-nuke to clean up named resources associated
+# with a specific managed cluster across an Azure account. CLUSTER_NAME is
+# typically the metadata.name of the ManagedCluster.
+# This will nuke the ResourceGroup affiliated with the ManagedCluster.
+#
+# Usage:
+# 'CLUSTER_NAME=foo AZURE_REGION=westus3 AZURE_TENANT_ID=12345 make dev-azure-nuke' 
+# 
+# Check cluster names with 'kubectl get managedcluster.hmc.mirantis.com -n hmc-system'
+
+regions:
+ - global
+ - ${AZURE_REGION}
+
+resource-types:
+ includes:
+ - ResourceGroup
+
+accounts:
+ ${AZURE_TENANT_ID}:
+ filters:
+ __global__:
+ - ResourceGroup:
+ type: "glob"
+ value: "${CLUSTER_NAME}*"
+ invert: true
diff --git a/test/e2e/managedcluster/constants.go b/test/e2e/managedcluster/constants.go
@@ -47,4 +47,5 @@ const (
  EnvVarAzureTenantID = "AZURE_TENANT_ID"
  EnvVarAzureSubscription = "AZURE_SUBSCRIPTION"
  EnvVarAzureClusterIdentity = "AZURE_CLUSTER_IDENTITY"
+ EnvVarAzureRegion = "AZURE_REGION"
 )
diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go
@@ -97,7 +97,7 @@ func setClusterName(templateName Template) {
  // different deployed ManagedClusters.
  generatedName = fmt.Sprintf("%s-%s", mcName, providerName)
  if strings.Contains(string(templateName), "hosted") {
- generatedName = fmt.Sprintf("%s-%s", generatedName, "hosted")
+ generatedName = fmt.Sprintf("%s-%s", mcName, "hosted")
  }
 
  GinkgoT().Setenv(EnvVarManagedClusterName, generatedName)

diff --git a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl
@@ -7,7 +7,7 @@ spec:
  template: azure-hosted-cp
  credential: ${AZURE_CLUSTER_IDENTITY}-cred
  config:
- location: "westus2"
+ location: "${AZURE_REGION}"
  subscriptionID: "${AZURE_SUBSCRIPTION_ID}"
  vmSize: Standard_A4_v2
  clusterIdentity:

diff --git a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl
@@ -9,7 +9,7 @@ spec:
  config:
  controlPlaneNumber: 1
  workersNumber: 1
- location: "westus2"
+ location: "${AZURE_REGION}"
  subscriptionID: "${AZURE_SUBSCRIPTION_ID}"
  controlPlane:
  vmSize: Standard_A4_v2

diff --git a/test/e2e/provider_vsphere_test.go b/test/e2e/provider_vsphere_test.go
@@ -59,6 +59,9 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere"
  // VSphere doesn't have any form of cleanup outside of reconciling a
  // cluster deletion so we need to keep the test active while we wait
  // for CAPV to clean up the resources.
+ // TODO: Add an exterior cleanup mechanism for VSphere like
+ // 'dev-aws-nuke' to clean up resources in the event that the test
+ // fails to do so.
  if deleteFunc != nil && !noCleanup() {
  deletionValidator := managedcluster.NewProviderValidator(
  managedcluster.TemplateVSphereStandaloneCP,
@@ -79,7 +82,7 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere"
  d := managedcluster.GetUnstructured(managedcluster.TemplateVSphereStandaloneCP)
  clusterName = d.GetName()
 
- deleteFunc := kc.CreateManagedCluster(context.Background(), d)
+ deleteFunc = kc.CreateManagedCluster(context.Background(), d)
 
  By("waiting for infrastructure providers to deploy successfully")
  deploymentValidator := managedcluster.NewProviderValidator(