refactor: Migrate network pkg to Azure SDK v2 - Part 3 (#424)

Signed-off-by: Heba Elayoty <hebaelayoty@gmail.com>
virtual-kubelet · Jan 24, 2023 · d17a4fd · d17a4fd
1 parent 80d7d19
commit d17a4fd
Show file tree

Hide file tree

Showing 3 changed files with 140 additions and 57 deletions.
diff --git a/go.mod b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0
 	github.com/Azure/go-autorest/autorest v0.11.24
 	github.com/Azure/go-autorest/autorest/adal v0.9.20
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11

diff --git a/go.sum b/go.sum
@@ -24,6 +24,10 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 h1:eY6fhA944YceJrJ9OGn1T5iqe5DA2rQ+O1/Gi3P4bXU=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1/go.mod h1:5Q/hN8CkM0y7bBldgIdoPMp9jyBZ1KVeexQvfY2KXw8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0 h1:mk57wRUA8fyjFxVcPPGv4shLcWDXPFYokTJL9zJxQtE=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0/go.mod h1:mU96hbp8qJDA9OzTV1Ji7wCyPyaqC5kI6ZPsZfJ8sE4=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=

diff --git a/pkg/network/aci_network.go b/pkg/network/aci_network.go
@@ -8,17 +8,20 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"strings"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
 	"github.com/pkg/errors"
 	"github.com/virtual-kubelet/azure-aci/pkg/util"
-	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 
 	azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2"
-	aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network"
+	aznetworkv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
 	"github.com/virtual-kubelet/azure-aci/pkg/auth"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	v1 "k8s.io/api/core/v1"
@@ -32,6 +35,12 @@ const (
 	subnetDelegationService = "Microsoft.ContainerInstance/containerGroups"
 )
 
+var (
+	delegationName = "aciDelegation"
+	serviceName    = "Microsoft.ContainerInstance/containerGroups"
+	subnetAction   = "Microsoft.Network/virtualNetworks/subnets/action"
+)
+
 type ProviderNetwork struct {
 	VnetSubscriptionID string
 	VnetName           string
@@ -100,79 +109,148 @@ func (pn *ProviderNetwork) SetVNETConfig(ctx context.Context, azConfig *auth.Con
 
 func (pn *ProviderNetwork) setupNetwork(ctx context.Context, azConfig *auth.Config) error {
 	logger := log.G(ctx).WithField("method", "setupNetwork")
-	logger.Debug("setting up network")
+	ctx, span := trace.StartSpan(ctx, "network.setupNetwork")
+	defer span.End()
 
-	c := aznetwork.NewSubnetsClient(azConfig.AuthConfig.SubscriptionID)
-	c.Authorizer = azConfig.Authorizer
+	subnetsClient, err := getSubnetClient(ctx, azConfig)
+	if err != nil {
+		return err
+	}
+
+	var rawResponse *http.Response
+	ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse)
 
 	createSubnet := true
-	subnet, err := c.Get(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, "")
-	if err != nil && !errdefs.IsNotFound(err) {
-		return fmt.Errorf("error while looking up subnet: %v", err)
-	}
-	if errdefs.IsNotFound(err) && pn.SubnetCIDR == "" {
-		return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", pn.SubnetName, pn.VnetName, pn.VnetResourceGroup, pn.VnetSubscriptionID)
-	}
-	if err == nil {
-		if pn.SubnetCIDR == "" {
-			pn.SubnetCIDR = *subnet.SubnetPropertiesFormat.AddressPrefix
-		}
-		if pn.SubnetCIDR != *subnet.SubnetPropertiesFormat.AddressPrefix {
-			return fmt.Errorf("found subnet '%s' using different CIDR: '%s'. desired: '%s'", pn.SubnetName, *subnet.SubnetPropertiesFormat.AddressPrefix, pn.SubnetCIDR)
+	response, err := subnetsClient.Get(ctxWithResp, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, nil)
+	var respErr *azcore.ResponseError
+	if err != nil {
+		if errors.As(err, &respErr) && !(respErr.RawResponse.StatusCode == http.StatusNotFound) {
+			return fmt.Errorf("error while looking up subnet: %v", err)
 		}
-		if subnet.SubnetPropertiesFormat.RouteTable != nil {
-			return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance since it references the route table '%s'", pn.SubnetName, *subnet.SubnetPropertiesFormat.RouteTable.ID)
+
+		if respErr.RawResponse.StatusCode == http.StatusNotFound && pn.SubnetCIDR == "" {
+			return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", pn.SubnetName, pn.VnetName, pn.VnetResourceGroup, pn.VnetSubscriptionID)
 		}
-		if subnet.SubnetPropertiesFormat.ServiceAssociationLinks != nil {
-			for _, l := range *subnet.SubnetPropertiesFormat.ServiceAssociationLinks {
-				if l.ServiceAssociationLinkPropertiesFormat != nil {
-					if *l.ServiceAssociationLinkPropertiesFormat.LinkedResourceType == subnetDelegationService {
+	}
+	currentSubnet := response.Subnet
+
+	if err == nil {
+		if currentSubnet.Properties.AddressPrefix != nil {
+			if pn.SubnetCIDR == "" {
+				pn.SubnetCIDR = *currentSubnet.Properties.AddressPrefix
+			}
+			if pn.SubnetCIDR != *currentSubnet.Properties.AddressPrefix {
+				return fmt.Errorf("found subnet '%s' using different CIDR: '%s'. desired: '%s'", pn.SubnetName, *currentSubnet.Properties.AddressPrefix, pn.SubnetCIDR)
+			}
+			if currentSubnet.Properties.RouteTable != nil {
+				return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance since it references the route table '%s'", pn.SubnetName, *currentSubnet.Properties.RouteTable.ID)
+			}
+			if currentSubnet.Properties.ServiceAssociationLinks != nil {
+				for _, l := range currentSubnet.Properties.ServiceAssociationLinks {
+					if l.Properties != nil && l.Properties.LinkedResourceType != nil {
+						if *l.Properties.LinkedResourceType == subnetDelegationService {
+							createSubnet = false
+							break
+						} else {
+							return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance as it is used by other Azure resource: '%v'", pn.SubnetName, l)
+						}
+					}
+				}
+			} else {
+				for _, d := range currentSubnet.Properties.Delegations {
+					if d.Properties != nil && d.Properties.ServiceName != nil &&
+						*d.Properties.ServiceName == subnetDelegationService {
 						createSubnet = false
 						break
-					} else {
-						return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance as it is used by other Azure resource: '%v'", pn.SubnetName, l)
 					}
 				}
 			}
-		} else {
-			for _, d := range *subnet.SubnetPropertiesFormat.Delegations {
-				if d.ServiceDelegationPropertiesFormat != nil && *d.ServiceDelegationPropertiesFormat.ServiceName == subnetDelegationService {
-					createSubnet = false
-					break
-				}
-			}
 		}
 	}
 
 	if createSubnet {
-		logger.Debug("creating a subnet")
-
-		var (
-			delegationName = "aciDelegation"
-			serviceName    = "Microsoft.ContainerInstance/containerGroups"
-			subnetAction   = "Microsoft.Network/virtualNetworks/subnets/action"
-		)
-
-		subnet = aznetwork.Subnet{
-			Name: &pn.SubnetName,
-			SubnetPropertiesFormat: &aznetwork.SubnetPropertiesFormat{
-				AddressPrefix: &pn.SubnetCIDR,
-				Delegations: &[]aznetwork.Delegation{
-					{
-						Name: &delegationName,
-						ServiceDelegationPropertiesFormat: &aznetwork.ServiceDelegationPropertiesFormat{
-							ServiceName: &serviceName,
-							Actions:     &[]string{subnetAction},
-						},
+		logger.Debugf("new subnet %s is creating", pn.SubnetName)
+
+		err2 := pn.createACISubnet(ctx, subnetsClient)
+		if err2 != nil {
+			return err2
+		}
+	}
+
+	logger.Debug("setup network is successful")
+	return nil
+}
+
+func getSubnetClient(ctx context.Context, azConfig *auth.Config) (*aznetworkv2.SubnetsClient, error) {
+	logger := log.G(ctx).WithField("method", "getSubnetClient")
+	ctx, span := trace.StartSpan(ctx, "network.getSubnetClient")
+	defer span.End()
+
+	logger.Debug("getting azure credential")
+
+	var err error
+	var credential azcore.TokenCredential
+	isUserIdentity := len(azConfig.AuthConfig.ClientID) == 0
+
+	if isUserIdentity {
+		credential, err = azConfig.GetMSICredential(ctx)
+	} else {
+		credential, err = azConfig.GetSPCredential(ctx)
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "an error has occurred while creating getting credential ")
+	}
+
+	options := arm.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Cloud: azConfig.Cloud,
+		},
+	}
+
+	subnetsClient, err := aznetworkv2.NewSubnetsClient(azConfig.AuthConfig.SubscriptionID, credential, &options)
+	if err != nil {
+		return nil, errors.Wrap(err, "an error has occurred while creating subnet client")
+	}
+	return subnetsClient, nil
+}
+
+// createACISubnet create new subnet for ACI
+func (pn *ProviderNetwork) createACISubnet(ctx context.Context, subnetsClient *aznetworkv2.SubnetsClient) error {
+	logger := log.G(ctx).WithField("method", "createACISubnet")
+	ctx, span := trace.StartSpan(ctx, "network.createACISubnet")
+	defer span.End()
+
+	logger.Debug("creating a subnet")
+
+	subnet := aznetworkv2.Subnet{
+		Name: &pn.SubnetName,
+		Properties: &aznetworkv2.SubnetPropertiesFormat{
+			AddressPrefix: &pn.SubnetCIDR,
+			Delegations: []*aznetworkv2.Delegation{
+				{
+					Name: &delegationName,
+					Properties: &aznetworkv2.ServiceDelegationPropertiesFormat{
+						ServiceName: &serviceName,
+						Actions:     []*string{&subnetAction},
 					},
 				},
 			},
-		}
-		_, err = c.CreateOrUpdate(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, subnet)
-		if err != nil {
-			return fmt.Errorf("error creating subnet: %v", err)
-		}
+		},
+	}
+
+	var rawResponse *http.Response
+	ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse)
+
+	poller, err := subnetsClient.BeginCreateOrUpdate(ctxWithResp, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, subnet, nil)
+	if err != nil {
+		return fmt.Errorf("error creating subnet: %v", err)
+	}
+	_, err = poller.PollUntilDone(context.TODO(), nil)
+	if err != nil {
+		return fmt.Errorf("error creating subnet: %v", err)
 	}
+	logger.Debugf("new subnet %s has been created successfully. vnet %s, response code %d", pn.SubnetName, pn.VnetName, rawResponse.StatusCode)
+	logger.Infof("new subnet %s has been created successfully", pn.SubnetName)
 	return nil
 }