use existing resource group

cmd/kola/options.go

@@ -123,6 +123,10 @@ func init() {
 	bv(&kola.AzureOptions.UseGallery, "azure-use-gallery", false, "Use gallery image instead of managed image")
 	bv(&kola.AzureOptions.UsePrivateIPs, "azure-use-private-ips", false, "Assume nodes are reachable using private IP addresses")
 	bv(&kola.AzureOptions.UseIdentity, "azure-identity", false, "Use VM managed identity for authentication (default false)")
+	sv(&kola.AzureOptions.ResourceGroup, "azure-resource-group", "", "Deploy resources in an existing resource group")
+	sv(&kola.AzureOptions.AvailabilitySetID, "azure-availability-set-id", "", "Deploy instances with an existing availibity set")
+	// TODO: Handle list of values.
+	sv(&kola.AzureOptions.ResourceToKeep, "azure-resource-to-keep", "Microsoft.Compute/availabilitySets", "Keep this resource when deleting resources in the given resource group")
 
 	// do-specific options
 	sv(&kola.DOOptions.ConfigPath, "do-config-file", "", "DigitalOcean config file (default \"~/"+auth.DOConfigPath+"\")")

platform/api/azure/api.go

@@ -38,23 +38,28 @@ import (
 	internalAuth "github.com/flatcar/mantle/auth"
 )
 
+const (
+	APIVersion = "2023-09-01"
+)
+
 var (
 	plog = capnslog.NewPackageLogger("github.com/flatcar/mantle", "platform/api/azure")
 )
 
 type API struct {
-	client      management.Client
-	rgClient    resources.GroupsClient
-	depClient   resources.DeploymentsClient
-	imgClient   compute.ImagesClient
-	compClient  compute.VirtualMachinesClient
-	vmImgClient compute.VirtualMachineImagesClient
-	netClient   network.VirtualNetworksClient
-	subClient   network.SubnetsClient
-	ipClient    network.PublicIPAddressesClient
-	intClient   network.InterfacesClient
-	accClient   armStorage.AccountsClient
-	Opts        *Options
+	client          management.Client
+	rgClient        resources.GroupsClient
+	depClient       resources.DeploymentsClient
+	resourcesClient resources.Client
+	imgClient       compute.ImagesClient
+	compClient      compute.VirtualMachinesClient
+	vmImgClient     compute.VirtualMachineImagesClient
+	netClient       network.VirtualNetworksClient
+	subClient       network.SubnetsClient
+	ipClient        network.PublicIPAddressesClient
+	intClient       network.InterfacesClient
+	accClient       armStorage.AccountsClient
+	Opts            *Options
 }
 
 type Network struct {
@@ -121,7 +126,7 @@ func setOptsFromProfile(opts *Options) error {
 func New(opts *Options) (*API, error) {
 	var err error
 	conf := management.DefaultConfig()
-	conf.APIVersion = "2015-04-01"
+	conf.APIVersion = APIVersion
 
 	if opts.ManagementURL != "" {
 		conf.ManagementURL = opts.ManagementURL
@@ -227,6 +232,9 @@ func (a *API) SetupClients() error {
 	a.depClient = resources.NewDeploymentsClient(subid)
 	a.depClient.Authorizer = auther
 
+	a.resourcesClient = resources.NewClient(subid)
+	a.resourcesClient.Authorizer = auther
+
 	auther, err = a.newAuthorizer(compute.DefaultBaseURI)
 	if err != nil {
 		return err
@@ -290,7 +298,7 @@ func (a *API) GC(gracePeriod time.Duration) error {
 				return fmt.Errorf("error parsing time: %v", err)
 			}
 			if !timeCreated.After(durationAgo) {
-				if err = a.TerminateResourceGroup(*l.Name); err != nil {
+				if err = a.TerminateResourceGroup(*l.Name, false); err != nil {
 					return err
 				}
 			}

platform/api/azure/groups.go

@@ -16,6 +16,7 @@ package azure
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2020-10-01/resources"
@@ -41,17 +42,45 @@ func (a *API) CreateResourceGroup(prefix string) (string, error) {
 	return name, nil
 }
 
-func (a *API) TerminateResourceGroup(name string) error {
-	resp, err := a.rgClient.CheckExistence(context.TODO(), name)
-	if err != nil {
+// keepResourceGroup can be used to terminate all the resources created in the group but keep the group itself.
+func (a *API) TerminateResourceGroup(name string, keepResourceGroup bool) error {
+	if !keepResourceGroup {
+		resp, err := a.rgClient.CheckExistence(context.TODO(), name)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode != 204 {
+			return nil
+		}
+
+		_, err = a.rgClient.Delete(context.TODO(), name)
 		return err
 	}
-	if resp.StatusCode != 204 {
-		return nil
+
+	// Get the list of resources to delete in the group
+	listResult, err := a.resourcesClient.ListByResourceGroup(context.TODO(), name, "", "", nil)
+	if err != nil {
+		return fmt.Errorf("listing by resource group: %v", err)
+	}
+
+	for _, value := range listResult.Values() {
+		id := *value.ID
+		t := *value.Type
+
+		if id != "" && t != "" {
+			if t == a.Opts.ResourceToKeep {
+				continue
+			}
+
+			if _, err := a.resourcesClient.DeleteByID(context.TODO(), id, APIVersion); err != nil {
+				return fmt.Errorf("deleting resource %s: %v", id, err)
+			}
+
+			plog.Infof("deleted collected resource: kind: %s", *value.Type)
+		}
 	}
 
-	_, err = a.rgClient.Delete(context.TODO(), name)
-	return err
+	return nil
 }
 
 func (a *API) ListResourceGroups(filter string) (resources.GroupListResult, error) {

platform/api/azure/instance.go

@@ -149,6 +149,11 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string,
 		}
 	}
 
+	availabilitySetID := a.Opts.AvailabilitySetID
+	if availabilitySetID != "" {
+		vm.VirtualMachineProperties.AvailabilitySet = &compute.SubResource{ID: &availabilitySetID}
+	}
+
 	return vm
 }
 

platform/api/azure/options.go

@@ -49,6 +49,12 @@ type Options struct {
 
 	// Azure Storage API endpoint suffix. If unset, the Azure SDK default will be used.
 	StorageEndpointSuffix string
-	// UseUserData can be use to enable custom data only or user-data only.
+	// UseUserData can be used to enable custom data only or user-data only.
 	UseUserData bool
+	// ResourceGroup is an existing resource group to deploy resources in.
+	ResourceGroup string
+	// AvailabilitySetID is an existing availability set to deploy the instance in.
+	AvailabilitySetID string
+	// ResourceToKeep is a resource to keep when cleaning an existing ResourceGroup.
+	ResourceToKeep string
 }

platform/machine/azure/cluster.go

@@ -90,7 +90,10 @@ func (ac *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error)
 func (ac *cluster) Destroy() {
 	ac.BaseCluster.Destroy()
 	if ac.ResourceGroup != ac.flight.ImageResourceGroup {
-		if e := ac.flight.Api.TerminateResourceGroup(ac.ResourceGroup); e != nil {
+		// If the resource group is provided via the command line, we need to delete the resources created inside
+		// but we keep the resource group itself.
+		keepResourceGroup := ac.flight.Api.Opts.ResourceGroup != ""
+		if e := ac.flight.Api.TerminateResourceGroup(ac.ResourceGroup, keepResourceGroup); e != nil {
 			plog.Errorf("Deleting resource group %v: %v", ac.ResourceGroup, e)
 		}
 	}

platform/machine/azure/flight.go

@@ -84,9 +84,15 @@ func NewFlight(opts *azure.Options) (platform.Flight, error) {
 		blobName := imageName + ".vhd"
 		container := "temp"
 
-		af.ImageResourceGroup, err = af.Api.CreateResourceGroup("kola-cluster-image")
-		if err != nil {
-			return nil, err
+		rg := opts.ResourceGroup
+		if rg != "" {
+			af.ImageResourceGroup = rg
+			plog.Infof("Using existing resource group: %s", rg)
+		} else {
+			af.ImageResourceGroup, err = af.Api.CreateResourceGroup("kola-cluster-image")
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		af.ImageStorageAccount, err = af.Api.CreateStorageAccount(af.ImageResourceGroup)
@@ -177,9 +183,15 @@ func (af *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 		ac.StorageAccount = af.ImageStorageAccount
 		ac.Network = af.Network
 	} else {
-		ac.ResourceGroup, err = af.Api.CreateResourceGroup("kola-cluster")
-		if err != nil {
-			return nil, err
+		rg := af.Api.Opts.ResourceGroup
+		if rg != "" {
+			ac.ResourceGroup = rg
+			plog.Infof("Using existing resource group: %s", rg)
+		} else {
+			ac.ResourceGroup, err = af.Api.CreateResourceGroup("kola-cluster")
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		ac.StorageAccount, err = af.Api.CreateStorageAccount(ac.ResourceGroup)
@@ -203,7 +215,10 @@ func (af *flight) Destroy() {
 	af.BaseFlight.Destroy()
 
 	if af.ImageResourceGroup != "" {
-		if e := af.Api.TerminateResourceGroup(af.ImageResourceGroup); e != nil {
+		// If the resource group is provided via the command line, we need to delete the resources created inside
+		// but we keep the resource group itself.
+		keepResourceGroup := af.Api.Opts.ResourceGroup != ""
+		if e := af.Api.TerminateResourceGroup(af.ImageResourceGroup, keepResourceGroup); e != nil {
 			plog.Errorf("Deleting image resource group %v: %v", af.ImageResourceGroup, e)
 		}
 	}