diff --git a/go.mod b/go.mod index 2227e686..5b0db4de 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0 github.com/Azure/go-autorest/autorest v0.11.24 github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 diff --git a/go.sum b/go.sum index 4e7ad4ab..d5e9a9cc 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,10 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5 github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 h1:eY6fhA944YceJrJ9OGn1T5iqe5DA2rQ+O1/Gi3P4bXU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1/go.mod h1:5Q/hN8CkM0y7bBldgIdoPMp9jyBZ1KVeexQvfY2KXw8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0 h1:mk57wRUA8fyjFxVcPPGv4shLcWDXPFYokTJL9zJxQtE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0/go.mod h1:mU96hbp8qJDA9OzTV1Ji7wCyPyaqC5kI6ZPsZfJ8sE4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= diff --git a/pkg/network/aci_network.go b/pkg/network/aci_network.go index 1657faea..7c46b97d 100644 --- a/pkg/network/aci_network.go +++ b/pkg/network/aci_network.go @@ -8,17 +8,20 @@ import ( "context" "fmt" "net" + "net/http" "os" "strings" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/pkg/util" - "github.com/virtual-kubelet/virtual-kubelet/errdefs" "github.com/virtual-kubelet/virtual-kubelet/trace" utilvalidation "k8s.io/apimachinery/pkg/util/validation" azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" - aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" + aznetworkv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" "github.com/virtual-kubelet/azure-aci/pkg/auth" "github.com/virtual-kubelet/virtual-kubelet/log" v1 "k8s.io/api/core/v1" @@ -32,6 +35,12 @@ const ( subnetDelegationService = "Microsoft.ContainerInstance/containerGroups" ) +var ( + delegationName = "aciDelegation" + serviceName = "Microsoft.ContainerInstance/containerGroups" + subnetAction = "Microsoft.Network/virtualNetworks/subnets/action" +) + type ProviderNetwork struct { VnetSubscriptionID string VnetName string @@ -100,79 +109,148 @@ func (pn *ProviderNetwork) SetVNETConfig(ctx context.Context, azConfig *auth.Con func (pn *ProviderNetwork) setupNetwork(ctx context.Context, azConfig *auth.Config) error { logger := log.G(ctx).WithField("method", "setupNetwork") - logger.Debug("setting up network") + ctx, span := trace.StartSpan(ctx, "network.setupNetwork") + defer span.End() - c := aznetwork.NewSubnetsClient(azConfig.AuthConfig.SubscriptionID) - c.Authorizer = azConfig.Authorizer + subnetsClient, err := getSubnetClient(ctx, azConfig) + if err != nil { + return err + } + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) createSubnet := true - subnet, err := c.Get(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, "") - if err != nil && !errdefs.IsNotFound(err) { - return fmt.Errorf("error while looking up subnet: %v", err) - } - if errdefs.IsNotFound(err) && pn.SubnetCIDR == "" { - return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", pn.SubnetName, pn.VnetName, pn.VnetResourceGroup, pn.VnetSubscriptionID) - } - if err == nil { - if pn.SubnetCIDR == "" { - pn.SubnetCIDR = *subnet.SubnetPropertiesFormat.AddressPrefix - } - if pn.SubnetCIDR != *subnet.SubnetPropertiesFormat.AddressPrefix { - return fmt.Errorf("found subnet '%s' using different CIDR: '%s'. desired: '%s'", pn.SubnetName, *subnet.SubnetPropertiesFormat.AddressPrefix, pn.SubnetCIDR) + response, err := subnetsClient.Get(ctxWithResp, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, nil) + var respErr *azcore.ResponseError + if err != nil { + if errors.As(err, &respErr) && !(respErr.RawResponse.StatusCode == http.StatusNotFound) { + return fmt.Errorf("error while looking up subnet: %v", err) } - if subnet.SubnetPropertiesFormat.RouteTable != nil { - return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance since it references the route table '%s'", pn.SubnetName, *subnet.SubnetPropertiesFormat.RouteTable.ID) + + if respErr.RawResponse.StatusCode == http.StatusNotFound && pn.SubnetCIDR == "" { + return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", pn.SubnetName, pn.VnetName, pn.VnetResourceGroup, pn.VnetSubscriptionID) } - if subnet.SubnetPropertiesFormat.ServiceAssociationLinks != nil { - for _, l := range *subnet.SubnetPropertiesFormat.ServiceAssociationLinks { - if l.ServiceAssociationLinkPropertiesFormat != nil { - if *l.ServiceAssociationLinkPropertiesFormat.LinkedResourceType == subnetDelegationService { + } + currentSubnet := response.Subnet + + if err == nil { + if currentSubnet.Properties.AddressPrefix != nil { + if pn.SubnetCIDR == "" { + pn.SubnetCIDR = *currentSubnet.Properties.AddressPrefix + } + if pn.SubnetCIDR != *currentSubnet.Properties.AddressPrefix { + return fmt.Errorf("found subnet '%s' using different CIDR: '%s'. desired: '%s'", pn.SubnetName, *currentSubnet.Properties.AddressPrefix, pn.SubnetCIDR) + } + if currentSubnet.Properties.RouteTable != nil { + return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance since it references the route table '%s'", pn.SubnetName, *currentSubnet.Properties.RouteTable.ID) + } + if currentSubnet.Properties.ServiceAssociationLinks != nil { + for _, l := range currentSubnet.Properties.ServiceAssociationLinks { + if l.Properties != nil && l.Properties.LinkedResourceType != nil { + if *l.Properties.LinkedResourceType == subnetDelegationService { + createSubnet = false + break + } else { + return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance as it is used by other Azure resource: '%v'", pn.SubnetName, l) + } + } + } + } else { + for _, d := range currentSubnet.Properties.Delegations { + if d.Properties != nil && d.Properties.ServiceName != nil && + *d.Properties.ServiceName == subnetDelegationService { createSubnet = false break - } else { - return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance as it is used by other Azure resource: '%v'", pn.SubnetName, l) } } } - } else { - for _, d := range *subnet.SubnetPropertiesFormat.Delegations { - if d.ServiceDelegationPropertiesFormat != nil && *d.ServiceDelegationPropertiesFormat.ServiceName == subnetDelegationService { - createSubnet = false - break - } - } } } if createSubnet { - logger.Debug("creating a subnet") - - var ( - delegationName = "aciDelegation" - serviceName = "Microsoft.ContainerInstance/containerGroups" - subnetAction = "Microsoft.Network/virtualNetworks/subnets/action" - ) - - subnet = aznetwork.Subnet{ - Name: &pn.SubnetName, - SubnetPropertiesFormat: &aznetwork.SubnetPropertiesFormat{ - AddressPrefix: &pn.SubnetCIDR, - Delegations: &[]aznetwork.Delegation{ - { - Name: &delegationName, - ServiceDelegationPropertiesFormat: &aznetwork.ServiceDelegationPropertiesFormat{ - ServiceName: &serviceName, - Actions: &[]string{subnetAction}, - }, + logger.Debugf("new subnet %s is creating", pn.SubnetName) + + err2 := pn.createACISubnet(ctx, subnetsClient) + if err2 != nil { + return err2 + } + } + + logger.Debug("setup network is successful") + return nil +} + +func getSubnetClient(ctx context.Context, azConfig *auth.Config) (*aznetworkv2.SubnetsClient, error) { + logger := log.G(ctx).WithField("method", "getSubnetClient") + ctx, span := trace.StartSpan(ctx, "network.getSubnetClient") + defer span.End() + + logger.Debug("getting azure credential") + + var err error + var credential azcore.TokenCredential + isUserIdentity := len(azConfig.AuthConfig.ClientID) == 0 + + if isUserIdentity { + credential, err = azConfig.GetMSICredential(ctx) + } else { + credential, err = azConfig.GetSPCredential(ctx) + } + if err != nil { + return nil, errors.Wrap(err, "an error has occurred while creating getting credential ") + } + + options := arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: azConfig.Cloud, + }, + } + + subnetsClient, err := aznetworkv2.NewSubnetsClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "an error has occurred while creating subnet client") + } + return subnetsClient, nil +} + +// createACISubnet create new subnet for ACI +func (pn *ProviderNetwork) createACISubnet(ctx context.Context, subnetsClient *aznetworkv2.SubnetsClient) error { + logger := log.G(ctx).WithField("method", "createACISubnet") + ctx, span := trace.StartSpan(ctx, "network.createACISubnet") + defer span.End() + + logger.Debug("creating a subnet") + + subnet := aznetworkv2.Subnet{ + Name: &pn.SubnetName, + Properties: &aznetworkv2.SubnetPropertiesFormat{ + AddressPrefix: &pn.SubnetCIDR, + Delegations: []*aznetworkv2.Delegation{ + { + Name: &delegationName, + Properties: &aznetworkv2.ServiceDelegationPropertiesFormat{ + ServiceName: &serviceName, + Actions: []*string{&subnetAction}, }, }, }, - } - _, err = c.CreateOrUpdate(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, subnet) - if err != nil { - return fmt.Errorf("error creating subnet: %v", err) - } + }, + } + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + poller, err := subnetsClient.BeginCreateOrUpdate(ctxWithResp, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, subnet, nil) + if err != nil { + return fmt.Errorf("error creating subnet: %v", err) + } + _, err = poller.PollUntilDone(context.TODO(), nil) + if err != nil { + return fmt.Errorf("error creating subnet: %v", err) } + logger.Debugf("new subnet %s has been created successfully. vnet %s, response code %d", pn.SubnetName, pn.VnetName, rawResponse.StatusCode) + logger.Infof("new subnet %s has been created successfully", pn.SubnetName) return nil }