diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index dbac2bbf..0de9458c 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -124,6 +124,7 @@ jobs: AWS_REGION: us-west-2 AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + AZURE_REGION: westus2 AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }} AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }} @@ -233,6 +234,9 @@ jobs: AWS_REGION: us-west-2 AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + AZURE_REGION: westus2 + AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }} CLUSTER_NAME: '${{ needs.build.outputs.clustername }}' run: | make dev-aws-nuke + make dev-azure-nuke diff --git a/.gitignore b/.gitignore index fe458609..be8549cc 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ go.work go.work.sum # cloud-nuke config -*cloud_nuke.yaml +*cloud-nuke.yaml # Test artifacts test/e2e/*.log diff --git a/Makefile b/Makefile index eba447c2..2f841125 100644 --- a/Makefile +++ b/Makefile @@ -205,8 +205,6 @@ REGISTRY_PORT ?= 5001 REGISTRY_REPO ?= oci://127.0.0.1:$(REGISTRY_PORT)/charts DEV_PROVIDER ?= aws REGISTRY_IS_OCI = $(shell echo $(REGISTRY_REPO) | grep -q oci && echo true || echo false) -CLUSTER_NAME ?= $(shell $(YQ) '.metadata.name' ./config/dev/deployment.yaml) - AWS_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS} ifndef ignore-not-found @@ -341,13 +339,23 @@ dev-mcluster-delete: envsubst dev-creds-apply: dev-$(DEV_PROVIDER)-creds .PHONY: dev-aws-nuke -dev-aws-nuke: envsubst awscli yq cloud-nuke ## Warning: Destructive! Nuke all AWS resources deployed by 'DEV_PROVIDER=aws dev-provider-apply', prefix with CLUSTER_NAME to nuke a specific cluster. +dev-aws-nuke: envsubst awscli yq cloud-nuke ## Warning: Destructive! Nuke all AWS resources deployed by 'DEV_PROVIDER=aws dev-mcluster-apply' @CLUSTER_NAME=$(CLUSTER_NAME) YQ=$(YQ) AWSCLI=$(AWSCLI) bash -c "./scripts/aws-nuke-ccm.sh elb" - @CLUSTER_NAME=$(CLUSTER_NAME) $(ENVSUBST) < config/dev/cloud_nuke.yaml.tpl > config/dev/cloud_nuke.yaml - DISABLE_TELEMETRY=true $(CLOUDNUKE) aws --region $$AWS_REGION --force --config config/dev/cloud_nuke.yaml --resource-type vpc,eip,nat-gateway,ec2,ec2-subnet,elb,elbv2,ebs,internet-gateway,network-interface,security-group - @rm config/dev/cloud_nuke.yaml + @CLUSTER_NAME=$(CLUSTER_NAME) $(ENVSUBST) < config/dev/aws-cloud-nuke.yaml.tpl > config/dev/aws-cloud-nuke.yaml + DISABLE_TELEMETRY=true $(CLOUDNUKE) aws --region $$AWS_REGION --force --config config/dev/aws-cloud-nuke.yaml --resource-type vpc,eip,nat-gateway,ec2,ec2-subnet,elb,elbv2,ebs,internet-gateway,network-interface,security-group + @rm config/dev/aws-cloud-nuke.yaml @CLUSTER_NAME=$(CLUSTER_NAME) YQ=$(YQ) AWSCLI=$(AWSCLI) bash -c "./scripts/aws-nuke-ccm.sh ebs" +.PHONY: dev-azure-nuke +dev-azure-nuke: envsubst azure-nuke ## Warning: Destructive! Nuke all Azure resources deployed by 'DEV_PROVIDER=azure dev-mcluster-apply' + @if [ "$(CLUSTER_NAME)" == "" ] || [ "$(AZURE_TENANT_ID)" == "" ] || [ "$(AZURE_REGION)" == "" ]; then \ + echo "CLUSTER_NAME, AZURE_TENANT_ID and AZURE_REGION must be set"; \ + exit 1; \ + fi + @CLUSTER_NAME=$(CLUSTER_NAME) $(ENVSUBST) < config/dev/azure-cloud-nuke.yaml.tpl > config/dev/azure-cloud-nuke.yaml + $(AZURENUKE) run --config config/dev/azure-cloud-nuke.yaml --force --no-dry-run + @rm config/dev/azure-cloud-nuke.yaml + .PHONY: cli-install cli-install: clusterawsadm clusterctl cloud-nuke envsubst yq awscli ## Install the necessary CLI tools for deployment, development and testing. @@ -379,6 +387,7 @@ YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION) CLUSTERAWSADM ?= $(LOCALBIN)/clusterawsadm CLUSTERCTL ?= $(LOCALBIN)/clusterctl CLOUDNUKE ?= $(LOCALBIN)/cloud-nuke +AZURENUKE ?= $(LOCALBIN)/azure-nuke ADDLICENSE ?= $(LOCALBIN)/addlicense-$(ADDLICENSE_VERSION) ENVSUBST ?= $(LOCALBIN)/envsubst-$(ENVSUBST_VERSION) AWSCLI ?= $(LOCALBIN)/aws @@ -391,6 +400,7 @@ HELM_VERSION ?= v3.15.1 KIND_VERSION ?= v0.23.0 YQ_VERSION ?= v4.44.2 CLOUDNUKE_VERSION = v0.37.1 +AZURENUKE_VERSION = v1.1.0 CLUSTERAWSADM_VERSION ?= v2.5.2 CLUSTERCTL_VERSION ?= v1.7.3 ADDLICENSE_VERSION ?= v1.1.1 @@ -450,6 +460,12 @@ $(CLOUDNUKE): | $(LOCALBIN) curl -sL https://github.com/gruntwork-io/cloud-nuke/releases/download/$(CLOUDNUKE_VERSION)/cloud-nuke_$(OS)_$(ARCH) -o $(CLOUDNUKE) chmod +x $(CLOUDNUKE) +.PHONY: azure-nuke +azure-nuke: $(AZURENUKE) ## Download azure-nuke locally if necessary. +$(AZURENUKE): | $(LOCALBIN) + curl -sL https://github.com/ekristen/azure-nuke/releases/download/$(AZURENUKE_VERSION)/azure-nuke-$(AZURENUKE_VERSION)-$(OS)-$(ARCH).tar.gz -o /tmp/azure-nuke.tar.gz + tar xvf /tmp/azure-nuke.tar.gz -C $(LOCALBIN) azure-nuke + .PHONY: clusterawsadm clusterawsadm: $(CLUSTERAWSADM) ## Download clusterawsadm locally if necessary. $(CLUSTERAWSADM): | $(LOCALBIN) diff --git a/config/dev/cloud_nuke.yaml.tpl b/config/dev/aws-cloud-nuke.yaml.tpl similarity index 96% rename from config/dev/cloud_nuke.yaml.tpl rename to config/dev/aws-cloud-nuke.yaml.tpl index 1888fe96..378bf20c 100644 --- a/config/dev/cloud_nuke.yaml.tpl +++ b/config/dev/aws-cloud-nuke.yaml.tpl @@ -1,14 +1,13 @@ # This config file is used by cloud-nuke to clean up named resources associated # with a specific managed cluster across an AWS account. CLUSTER_NAME is -# typically the metadata.name of the Deployment. +# typically the metadata.name of the ManagedCluster. # The resources listed here are ALL of the potential resources that can be # filtered by cloud-nuke, except for IAM resources since we'll never touch those. # See: https://github.com/gruntwork-io/cloud-nuke?tab=readme-ov-file#whats-supported # # Usage: -# - 'make aws-dev-nuke' will nuke resources affiliated with config/dev/deployment.yaml -# - 'CLUSTER_NAME=foo make aws-dev-nuke' will nuke resources affiliated with an AWS cluster named 'foo' -# Check cluster names with 'kubectl get clusters -n hmc-system' +# - 'CLUSTER_NAME=foo make dev-aws-nuke' will nuke resources affiliated with an AWS cluster named 'foo' +# Check cluster names with 'kubectl get managedcluster.hmc.mirantis.com -n hmc-system' ACM: include: diff --git a/config/dev/azure-cloud-nuke.yaml.tpl b/config/dev/azure-cloud-nuke.yaml.tpl new file mode 100644 index 00000000..be98b051 --- /dev/null +++ b/config/dev/azure-cloud-nuke.yaml.tpl @@ -0,0 +1,26 @@ +# This config file is used by azure-nuke to clean up named resources associated +# with a specific managed cluster across an Azure account. CLUSTER_NAME is +# typically the metadata.name of the ManagedCluster. +# This will nuke the ResourceGroup affiliated with the ManagedCluster. +# +# Usage: +# 'CLUSTER_NAME=foo AZURE_REGION=westus3 AZURE_TENANT_ID=12345 make dev-azure-nuke' +# +# Check cluster names with 'kubectl get managedcluster.hmc.mirantis.com -n hmc-system' + +regions: + - global + - ${AZURE_REGION} + +resource-types: + includes: + - ResourceGroup + +accounts: + ${AZURE_TENANT_ID}: + filters: + __global__: + - ResourceGroup: + type: "glob" + value: "${CLUSTER_NAME}*" + invert: true diff --git a/test/e2e/managedcluster/constants.go b/test/e2e/managedcluster/constants.go index e6941e4b..d5c1f057 100644 --- a/test/e2e/managedcluster/constants.go +++ b/test/e2e/managedcluster/constants.go @@ -47,4 +47,5 @@ const ( EnvVarAzureTenantID = "AZURE_TENANT_ID" EnvVarAzureSubscription = "AZURE_SUBSCRIPTION" EnvVarAzureClusterIdentity = "AZURE_CLUSTER_IDENTITY" + EnvVarAzureRegion = "AZURE_REGION" ) diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go index b1cd4562..2b849571 100644 --- a/test/e2e/managedcluster/managedcluster.go +++ b/test/e2e/managedcluster/managedcluster.go @@ -97,7 +97,7 @@ func setClusterName(templateName Template) { // different deployed ManagedClusters. generatedName = fmt.Sprintf("%s-%s", mcName, providerName) if strings.Contains(string(templateName), "hosted") { - generatedName = fmt.Sprintf("%s-%s", generatedName, "hosted") + generatedName = fmt.Sprintf("%s-%s", mcName, "hosted") } GinkgoT().Setenv(EnvVarManagedClusterName, generatedName) diff --git a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl index bae1c3a0..43de6326 100644 --- a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl @@ -7,7 +7,7 @@ spec: template: azure-hosted-cp credential: ${AZURE_CLUSTER_IDENTITY}-cred config: - location: "westus2" + location: "${AZURE_REGION}" subscriptionID: "${AZURE_SUBSCRIPTION_ID}" vmSize: Standard_A4_v2 clusterIdentity: diff --git a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl index 8a8201ce..f40a3678 100644 --- a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl @@ -9,7 +9,7 @@ spec: config: controlPlaneNumber: 1 workersNumber: 1 - location: "westus2" + location: "${AZURE_REGION}" subscriptionID: "${AZURE_SUBSCRIPTION_ID}" controlPlane: vmSize: Standard_A4_v2 diff --git a/test/e2e/provider_vsphere_test.go b/test/e2e/provider_vsphere_test.go index 2c003f09..613ab1cf 100644 --- a/test/e2e/provider_vsphere_test.go +++ b/test/e2e/provider_vsphere_test.go @@ -59,6 +59,9 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" // VSphere doesn't have any form of cleanup outside of reconciling a // cluster deletion so we need to keep the test active while we wait // for CAPV to clean up the resources. + // TODO: Add an exterior cleanup mechanism for VSphere like + // 'dev-aws-nuke' to clean up resources in the event that the test + // fails to do so. if deleteFunc != nil && !noCleanup() { deletionValidator := managedcluster.NewProviderValidator( managedcluster.TemplateVSphereStandaloneCP, @@ -79,7 +82,7 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" d := managedcluster.GetUnstructured(managedcluster.TemplateVSphereStandaloneCP) clusterName = d.GetName() - deleteFunc := kc.CreateManagedCluster(context.Background(), d) + deleteFunc = kc.CreateManagedCluster(context.Background(), d) By("waiting for infrastructure providers to deploy successfully") deploymentValidator := managedcluster.NewProviderValidator(