Support upgrading a Kubernetes cluster that contains Virtual Machine Scale Sets

pkg/armhelpers/azureclient.go

@@ -46,17 +46,18 @@ type AzureClient struct {
 	environment     azure.Environment
 	subscriptionID  string
 
-	authorizationClient           authorization.RoleAssignmentsClient
-	deploymentsClient             resources.DeploymentsClient
-	deploymentOperationsClient    resources.DeploymentOperationsClient
-	resourcesClient               resources.GroupClient
-	storageAccountsClient         storage.AccountsClient
-	interfacesClient              network.InterfacesClient
-	groupsClient                  resources.GroupsClient
-	providersClient               resources.ProvidersClient
-	virtualMachinesClient         compute.VirtualMachinesClient
-	virtualMachineScaleSetsClient compute.VirtualMachineScaleSetsClient
-	disksClient                   disk.DisksClient
+	authorizationClient             authorization.RoleAssignmentsClient
+	deploymentsClient               resources.DeploymentsClient
+	deploymentOperationsClient      resources.DeploymentOperationsClient
+	resourcesClient                 resources.GroupClient
+	storageAccountsClient           storage.AccountsClient
+	interfacesClient                network.InterfacesClient
+	groupsClient                    resources.GroupsClient
+	providersClient                 resources.ProvidersClient
+	virtualMachinesClient           compute.VirtualMachinesClient
+	virtualMachineScaleSetsClient   compute.VirtualMachineScaleSetsClient
+	virtualMachineScaleSetVMsClient compute.VirtualMachineScaleSetVMsClient
+	disksClient                     disk.DisksClient
 
 	applicationsClient      graphrbac.ApplicationsClient
 	servicePrincipalsClient graphrbac.ServicePrincipalsClient
@@ -264,17 +265,18 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armSpt *a
 		environment:    env,
 		subscriptionID: subscriptionID,
 
-		authorizationClient:           authorization.NewRoleAssignmentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		deploymentsClient:             resources.NewDeploymentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		deploymentOperationsClient:    resources.NewDeploymentOperationsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		resourcesClient:               resources.NewGroupClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		storageAccountsClient:         storage.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		interfacesClient:              network.NewInterfacesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		groupsClient:                  resources.NewGroupsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		providersClient:               resources.NewProvidersClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		virtualMachinesClient:         compute.NewVirtualMachinesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		virtualMachineScaleSetsClient: compute.NewVirtualMachineScaleSetsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
-		disksClient:                   disk.NewDisksClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		authorizationClient:             authorization.NewRoleAssignmentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		deploymentsClient:               resources.NewDeploymentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		deploymentOperationsClient:      resources.NewDeploymentOperationsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		resourcesClient:                 resources.NewGroupClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		storageAccountsClient:           storage.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		interfacesClient:                network.NewInterfacesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		groupsClient:                    resources.NewGroupsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		providersClient:                 resources.NewProvidersClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		virtualMachinesClient:           compute.NewVirtualMachinesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		virtualMachineScaleSetsClient:   compute.NewVirtualMachineScaleSetsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		virtualMachineScaleSetVMsClient: compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
+		disksClient:                     disk.NewDisksClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
 
 		applicationsClient:      graphrbac.NewApplicationsClientWithBaseURI(env.GraphEndpoint, tenantID),
 		servicePrincipalsClient: graphrbac.NewServicePrincipalsClientWithBaseURI(env.GraphEndpoint, tenantID),
@@ -291,6 +293,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armSpt *a
 	c.providersClient.Authorizer = authorizer
 	c.virtualMachinesClient.Authorizer = authorizer
 	c.virtualMachineScaleSetsClient.Authorizer = authorizer
+	c.virtualMachineScaleSetVMsClient.Authorizer = authorizer
 	c.disksClient.Authorizer = authorizer
 
 	c.deploymentsClient.PollingDelay = time.Second * 5

pkg/armhelpers/compute.go

@@ -23,3 +23,26 @@ func (az *AzureClient) DeleteVirtualMachine(resourceGroup, name string, cancel <
 func (az *AzureClient) ListVirtualMachineScaleSets(resourceGroup string) (compute.VirtualMachineScaleSetListResult, error) {
 	return az.virtualMachineScaleSetsClient.List(resourceGroup)
 }
+
+// ListVirtualMachineScaleSetVMs returns the list of VMs per VMSS
+func (az *AzureClient) ListVirtualMachineScaleSetVMs(resourceGroup, virtualMachineScaleSet string) (compute.VirtualMachineScaleSetVMListResult, error) {
+	return az.virtualMachineScaleSetVMsClient.List(resourceGroup, virtualMachineScaleSet, "", "", "")
+}
+
+// DeleteVirtualMachineScaleSetVM deletes a VM in a VMSS
+func (az *AzureClient) DeleteVirtualMachineScaleSetVM(resourceGroup, virtualMachineScaleSet, instanceID string, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error) {
+	return az.virtualMachineScaleSetVMsClient.Delete(resourceGroup, virtualMachineScaleSet, instanceID, cancel)
+}
+
+// SetVirtualMachineScaleSetCapacity sets the VMSS capacity
+func (az *AzureClient) SetVirtualMachineScaleSetCapacity(resourceGroup, virtualMachineScaleSet string, sku compute.Sku, location string, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error) {
+	return az.virtualMachineScaleSetsClient.CreateOrUpdate(
+		resourceGroup,
+		virtualMachineScaleSet,
+		compute.VirtualMachineScaleSet{
+			Location: &location,
+			Sku:      &sku,
+		},
+		cancel,
+	)
+}

pkg/armhelpers/interfaces.go

@@ -44,6 +44,15 @@ type ACSEngineClient interface {
 	// ListVirtualMachineScaleSets lists the vmss resources in the resource group
 	ListVirtualMachineScaleSets(resourceGroup string) (compute.VirtualMachineScaleSetListResult, error)
 
+	// ListVirtualMachineScaleSetVMs lists the virtual machines contained in a vmss
+	ListVirtualMachineScaleSetVMs(resourceGroup, virtualMachineScaleSet string) (compute.VirtualMachineScaleSetVMListResult, error)
+
+	// DeleteVirtualMachineScaleSetVM deletes a VM in a VMSS
+	DeleteVirtualMachineScaleSetVM(resourceGroup, virtualMachineScaleSet, instanceID string, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error)
+
+	// SetVirtualMachineScaleSetCapacity sets the VMSS capacity
+	SetVirtualMachineScaleSetCapacity(resourceGroup, virtualMachineScaleSet string, sku compute.Sku, location string, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error)
+
 	//
 	// STORAGE
 

pkg/armhelpers/mockclients.go

@@ -20,21 +20,24 @@ import (
 
 //MockACSEngineClient is an implementation of ACSEngineClient where all requests error out
 type MockACSEngineClient struct {
-	FailDeployTemplate              bool
-	FailDeployTemplateQuota         bool
-	FailDeployTemplateConflict      bool
-	FailEnsureResourceGroup         bool
-	FailListVirtualMachines         bool
-	FailListVirtualMachineScaleSets bool
-	FailGetVirtualMachine           bool
-	FailDeleteVirtualMachine        bool
-	FailGetStorageClient            bool
-	FailDeleteNetworkInterface      bool
-	FailGetKubernetesClient         bool
-	FailListProviders               bool
-	ShouldSupportVMIdentity         bool
-	FailDeleteRoleAssignment        bool
-	MockKubernetesClient            *MockKubernetesClient
+	FailDeployTemplate                    bool
+	FailDeployTemplateQuota               bool
+	FailDeployTemplateConflict            bool
+	FailEnsureResourceGroup               bool
+	FailListVirtualMachines               bool
+	FailListVirtualMachineScaleSets       bool
+	FailGetVirtualMachine                 bool
+	FailDeleteVirtualMachine              bool
+	FailDeleteVirtualMachineScaleSetVM    bool
+	FailSetVirtualMachineScaleSetCapacity bool
+	FailListVirtualMachineScaleSetVMs     bool
+	FailGetStorageClient                  bool
+	FailDeleteNetworkInterface            bool
+	FailGetKubernetesClient               bool
+	FailListProviders                     bool
+	ShouldSupportVMIdentity               bool
+	FailDeleteRoleAssignment              bool
+	MockKubernetesClient                  *MockKubernetesClient
 }
 
 //MockStorageClient mock implementation of StorageClient
@@ -344,6 +347,79 @@ func (mc *MockACSEngineClient) DeleteVirtualMachine(resourceGroup, name string,
 	return respChan, errChan
 }
 
+//DeleteVirtualMachineScaleSetVM mock
+func (mc *MockACSEngineClient) DeleteVirtualMachineScaleSetVM(resourceGroup, virtualMachineScaleSet, instanceID string, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error) {
+	if mc.FailDeleteVirtualMachineScaleSetVM {
+		errChan := make(chan error)
+		respChan := make(chan compute.OperationStatusResponse)
+		go func() {
+			defer func() {
+				close(errChan)
+			}()
+			defer func() {
+				close(respChan)
+			}()
+			errChan <- fmt.Errorf("DeleteVirtualMachineScaleSetVM failed")
+		}()
+		return respChan, errChan
+	}
+
+	errChan := make(chan error)
+	respChan := make(chan compute.OperationStatusResponse)
+	go func() {
+		defer func() {
+			close(errChan)
+		}()
+		defer func() {
+			close(respChan)
+		}()
+		errChan <- nil
+		respChan <- compute.OperationStatusResponse{}
+	}()
+	return respChan, errChan
+}
+
+//SetVirtualMachineScaleSetCapacity mock
+func (mc *MockACSEngineClient) SetVirtualMachineScaleSetCapacity(resourceGroup, virtualMachineScaleSet string, sku compute.Sku, location string, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error) {
+	if mc.FailSetVirtualMachineScaleSetCapacity {
+		errChan := make(chan error)
+		respChan := make(chan compute.VirtualMachineScaleSet)
+		go func() {
+			defer func() {
+				close(errChan)
+			}()
+			defer func() {
+				close(respChan)
+			}()
+			errChan <- fmt.Errorf("SetVirtualMachineScaleSetCapacity failed")
+		}()
+		return respChan, errChan
+	}
+
+	errChan := make(chan error)
+	respChan := make(chan compute.VirtualMachineScaleSet)
+	go func() {
+		defer func() {
+			close(errChan)
+		}()
+		defer func() {
+			close(respChan)
+		}()
+		errChan <- nil
+		respChan <- compute.VirtualMachineScaleSet{}
+	}()
+	return respChan, errChan
+}
+
+//ListVirtualMachineScaleSetVMs mock
+func (mc *MockACSEngineClient) ListVirtualMachineScaleSetVMs(resourceGroup, virtualMachineScaleSet string) (compute.VirtualMachineScaleSetVMListResult, error) {
+	if mc.FailDeleteVirtualMachineScaleSetVM {
+		return compute.VirtualMachineScaleSetVMListResult{}, fmt.Errorf("DeleteVirtualMachineScaleSetVM failed")
+	}
+
+	return compute.VirtualMachineScaleSetVMListResult{}, nil
+}
+
 //GetStorageClient mock
 func (mc *MockACSEngineClient) GetStorageClient(resourceGroup, accountName string) (ACSStorageClient, error) {
 	if mc.FailGetStorageClient {

pkg/operations/kubernetesupgrade/upgradecluster.go

@@ -27,10 +27,26 @@ type ClusterTopology struct {
 	AgentPoolsToUpgrade map[string]bool
 	AgentPools          map[string]*AgentPoolTopology
 
+	AgentPoolScaleSetsToUpgrade []AgentPoolScaleSet
+
 	MasterVMs         *[]compute.VirtualMachine
 	UpgradedMasterVMs *[]compute.VirtualMachine
 }
 
+// AgentPoolScaleSet contains necessary data required to upgrade a VMSS
+type AgentPoolScaleSet struct {
+	Name         string
+	Sku          compute.Sku
+	Location     string
+	VMsToUpgrade []AgentPoolScaleSetVM
+}
+
+// AgentPoolScaleSetVM represents a VM in a VMSS
+type AgentPoolScaleSetVM struct {
+	Name       string
+	InstanceID string
+}
+
 // AgentPoolTopology contains agent VMs in a single pool
 type AgentPoolTopology struct {
 	Identifier       *string
@@ -128,6 +144,50 @@ func (uc *UpgradeCluster) getClusterNodeStatus(subscriptionID uuid.UUID, resourc
 	targetOrchestratorTypeVersion := fmt.Sprintf("%s:%s", uc.DataModel.Properties.OrchestratorProfile.OrchestratorType,
 		uc.DataModel.Properties.OrchestratorProfile.OrchestratorVersion)
 
+	vmScaleSets, err := uc.Client.ListVirtualMachineScaleSets(resourceGroup)
+	if err != nil {
+		return err
+	}
+	if vmScaleSets.Value != nil {
+		for _, vmScaleSet := range *vmScaleSets.Value {
+			vmScaleSetVMs, err := uc.Client.ListVirtualMachineScaleSetVMs(resourceGroup, *vmScaleSet.Name)
+			if err != nil {
+				return err
+			}
+			scaleSetToUpgrade := AgentPoolScaleSet{
+				Name:     *vmScaleSet.Name,
+				Sku:      *vmScaleSet.Sku,
+				Location: *vmScaleSet.Location,
+			}
+			for _, vm := range *vmScaleSetVMs.Value {
+				if vm.Tags == nil || (*vm.Tags)["orchestrator"] == nil {
+					uc.Logger.Infof("No tags found for scale set VM: %s skipping.\n", *vm.Name)
+					continue
+				}
+
+				scaleSetVMOrchestratorTypeAndVersion := *(*vm.Tags)["orchestrator"]
+				if scaleSetVMOrchestratorTypeAndVersion != targetOrchestratorTypeVersion {
+					// This condition is a scale set VM that is an older version and should be handled
+					uc.Logger.Infof(
+						"VM %s in VMSS %s has a current tag of %s and a desired tag of %s. Upgrading this node.\n",
+						*vm.Name,
+						*vmScaleSet.Name,
+						scaleSetVMOrchestratorTypeAndVersion,
+						targetOrchestratorTypeVersion,
+					)
+					scaleSetToUpgrade.VMsToUpgrade = append(
+						scaleSetToUpgrade.VMsToUpgrade,
+						AgentPoolScaleSetVM{
+							Name:       *vm.VirtualMachineScaleSetVMProperties.OsProfile.ComputerName,
+							InstanceID: *vm.InstanceID,
+						},
+					)
+				}
+			}
+			uc.AgentPoolScaleSetsToUpgrade = append(uc.AgentPoolScaleSetsToUpgrade, scaleSetToUpgrade)
+		}
+	}
+
 	for _, vm := range *vmListResult.Value {
 		if vm.Tags == nil || (*vm.Tags)["orchestrator"] == nil {
 			uc.Logger.Infof("No tags found for VM: %s skipping.\n", *vm.Name)