From d8b6d2385aba969f4bf9908a517294dfd84bcb28 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Tue, 16 Feb 2021 17:05:50 -0800 Subject: [PATCH 01/11] Initial Implementation Signed-off-by: Eytan Avisror --- controllers/instancegroup_controller.go | 7 +- controllers/providers/aws/predicates.go | 18 ++ .../providers/kubernetes/rollingupdate.go | 88 ----- controllers/providers/kubernetes/utils.go | 27 +- controllers/provisioners/eks/eks.go | 6 +- controllers/provisioners/eks/eks_test.go | 2 +- controllers/provisioners/eks/rollingupdate.go | 302 ++++++++++++++++++ controllers/provisioners/eks/upgrade.go | 111 +------ controllers/provisioners/provisioners.go | 3 + go.mod | 3 +- go.sum | 94 +++--- main.go | 1 + 12 files changed, 409 insertions(+), 253 deletions(-) delete mode 100644 controllers/providers/kubernetes/rollingupdate.go create mode 100644 controllers/provisioners/eks/rollingupdate.go diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go index 0650f972..b3006f7e 100644 --- a/controllers/instancegroup_controller.go +++ b/controllers/instancegroup_controller.go @@ -52,6 +52,7 @@ type InstanceGroupReconciler struct { ConfigMap *corev1.ConfigMap Namespaces map[string]corev1.Namespace NamespacesLock *sync.Mutex + DrainGroups *sync.Map ConfigRetention int } @@ -123,6 +124,7 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque // set/unset finalizer r.SetFinalizer(instanceGroup) + namespacedName := instanceGroup.NamespacedName() input := provisioners.ProvisionerInput{ AwsWorker: r.Auth.Aws, @@ -139,6 +141,9 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque ) status.SetConfigHash(configHash) + drainGroup, _ := r.DrainGroups.LoadOrStore(namespacedName, &sync.WaitGroup{}) + input.DrainGroup = drainGroup.(*sync.WaitGroup) + if !reflect.DeepEqual(*r.ConfigMap, corev1.ConfigMap{}) { // Configmap exist - apply defaults/boundaries if namespace is not excluded namespace := instanceGroup.GetNamespace() @@ -150,7 +155,7 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque } if err = defaultConfig.SetDefaults(); err != nil { - r.Log.Error(err, "failed to set configuration defaults", "instancegroup", instanceGroup.NamespacedName()) + r.Log.Error(err, "failed to set configuration defaults", "instancegroup", namespacedName) return ctrl.Result{}, err } diff --git a/controllers/providers/aws/predicates.go b/controllers/providers/aws/predicates.go index 66647491..a15790a5 100644 --- a/controllers/providers/aws/predicates.go +++ b/controllers/providers/aws/predicates.go @@ -16,6 +16,8 @@ limitations under the License. package aws import ( + "strings" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/eks" @@ -98,3 +100,19 @@ func IsUsingMixedInstances(group *autoscaling.Group) bool { } return false } + +func IsDesiredInService(scalingGroup *autoscaling.Group) bool { + var ( + desired = aws.Int64Value(scalingGroup.DesiredCapacity) + inServiceCount int64 + ) + + for _, instance := range scalingGroup.Instances { + lifecycle := aws.StringValue(instance.LifecycleState) + if strings.EqualFold(lifecycle, autoscaling.LifecycleStateInService) { + inServiceCount++ + } + } + + return desired == inServiceCount +} diff --git a/controllers/providers/kubernetes/rollingupdate.go b/controllers/providers/kubernetes/rollingupdate.go deleted file mode 100644 index e0346ea9..00000000 --- a/controllers/providers/kubernetes/rollingupdate.go +++ /dev/null @@ -1,88 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubernetes - -import ( - awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" - corev1 "k8s.io/api/core/v1" - ctrl "sigs.k8s.io/controller-runtime" -) - -const ( - RollingUpdateStrategyName = "rollingupdate" -) - -var ( - log = ctrl.Log.WithName("kubernetes-provider") -) - -type RollingUpdateRequest struct { - AwsWorker awsprovider.AwsWorker - ClusterNodes *corev1.NodeList - ScalingGroupName string - MaxUnavailable int - DesiredCapacity int - AllInstances []string - UpdateTargets []string -} - -func ProcessRollingUpgradeStrategy(req *RollingUpdateRequest) (bool, error) { - - log.Info("starting rolling update", - "scalinggroup", req.ScalingGroupName, - "targets", req.UpdateTargets, - "maxunavailable", req.MaxUnavailable, - ) - if len(req.UpdateTargets) == 0 { - log.Info("no updatable instances", "scalinggroup", req.ScalingGroupName) - return true, nil - } - - // cannot rotate if maxUnavailable is greater than number of desired - if req.MaxUnavailable > req.DesiredCapacity { - log.Info("maxUnavailable exceeds desired capacity, setting maxUnavailable match desired", - "scalinggroup", req.ScalingGroupName, - "maxunavailable", req.MaxUnavailable, - "desiredcapacity", req.DesiredCapacity, - ) - req.MaxUnavailable = req.DesiredCapacity - } - - ok, err := IsMinNodesReady(req.ClusterNodes, req.AllInstances, req.MaxUnavailable) - if err != nil { - return false, err - } - - if !ok { - log.Info("desired nodes are not ready", "scalinggroup", req.ScalingGroupName) - return false, nil - } - - var terminateTargets []string - if req.MaxUnavailable <= len(req.UpdateTargets) { - terminateTargets = req.UpdateTargets[:req.MaxUnavailable] - } else { - terminateTargets = req.UpdateTargets - } - - log.Info("terminating targets", "scalinggroup", req.ScalingGroupName, "targets", terminateTargets) - if err := req.AwsWorker.TerminateScalingInstances(terminateTargets); err != nil { - // terminate failures are retryable - log.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", req.ScalingGroupName, "targets", terminateTargets) - return false, nil - } - return false, nil -} diff --git a/controllers/providers/kubernetes/utils.go b/controllers/providers/kubernetes/utils.go index 7756938a..c21c3428 100644 --- a/controllers/providers/kubernetes/utils.go +++ b/controllers/providers/kubernetes/utils.go @@ -40,9 +40,14 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) +var ( + log = ctrl.Log.WithName("kubernetes-provider") +) + type KubernetesClientSet struct { Kubernetes kubernetes.Interface KubeDynamic dynamic.Interface @@ -63,7 +68,7 @@ func IsDesiredNodesReady(nodes *corev1.NodeList, instanceIds []string, desiredCo return false, nil } - readyInstances := GetReadyNodesByInstance(instanceIds, nodes) + readyInstances := GetReadyNodeNamesByInstance(instanceIds, nodes) // if discovered nodes match provided instance ids, condition is ready if common.StringSliceEquals(readyInstances, instanceIds) { @@ -79,7 +84,7 @@ func IsMinNodesReady(nodes *corev1.NodeList, instanceIds []string, minCount int) return false, nil } - readyInstances := GetReadyNodesByInstance(instanceIds, nodes) + readyInstances := GetReadyNodeNamesByInstance(instanceIds, nodes) // if discovered nodes match provided instance ids, condition is ready if common.StringSliceContains(readyInstances, instanceIds) { @@ -89,7 +94,7 @@ func IsMinNodesReady(nodes *corev1.NodeList, instanceIds []string, minCount int) return false, nil } -func GetReadyNodesByInstance(instanceIds []string, nodes *corev1.NodeList) []string { +func GetReadyNodeNamesByInstance(instanceIds []string, nodes *corev1.NodeList) []string { readyInstances := make([]string, 0) for _, id := range instanceIds { for _, node := range nodes.Items { @@ -101,6 +106,22 @@ func GetReadyNodesByInstance(instanceIds []string, nodes *corev1.NodeList) []str return readyInstances } +func GetNodesByInstance(instanceIds []string, nodes *corev1.NodeList) *corev1.NodeList { + nodeList := &corev1.NodeList{ + ListMeta: metav1.ListMeta{}, + Items: []corev1.Node{}, + } + + for _, id := range instanceIds { + for _, node := range nodes.Items { + if common.GetLastElementBy(node.Spec.ProviderID, "/") == id { + nodeList.Items = append(nodeList.Items, node) + } + } + } + return nodeList +} + func IsNodeReady(n corev1.Node) bool { for _, condition := range n.Status.Conditions { if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go index 233f66b1..7d0cef0c 100644 --- a/controllers/provisioners/eks/eks.go +++ b/controllers/provisioners/eks/eks.go @@ -19,14 +19,12 @@ import ( "fmt" "sync" - corev1 "k8s.io/api/core/v1" - "github.com/go-logr/logr" - "github.com/keikoproj/instance-manager/api/v1alpha1" awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" "github.com/keikoproj/instance-manager/controllers/provisioners" + corev1 "k8s.io/api/core/v1" ) const ( @@ -65,6 +63,7 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext { ctx := &EksInstanceGroupContext{ InstanceGroup: instanceGroup, + DrainGroup: p.DrainGroup, KubernetesClient: p.Kubernetes, AwsWorker: p.AwsWorker, Log: p.Log.WithName("eks"), @@ -81,6 +80,7 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext { type EksInstanceGroupContext struct { sync.Mutex + DrainGroup *sync.WaitGroup InstanceGroup *v1alpha1.InstanceGroup KubernetesClient kubeprovider.KubernetesClientSet AwsWorker awsprovider.AwsWorker diff --git a/controllers/provisioners/eks/eks_test.go b/controllers/provisioners/eks/eks_test.go index 3f81cc40..fbd34f5d 100644 --- a/controllers/provisioners/eks/eks_test.go +++ b/controllers/provisioners/eks/eks_test.go @@ -381,7 +381,7 @@ func MockAwsCRDStrategy(spec string) v1alpha1.AwsUpgradeStrategy { func MockAwsRollingUpdateStrategy(maxUnavailable *intstr.IntOrString) v1alpha1.AwsUpgradeStrategy { return v1alpha1.AwsUpgradeStrategy{ - Type: kubeprovider.RollingUpdateStrategyName, + Type: RollingUpdateStrategyName, RollingUpdateType: &v1alpha1.RollingUpdateStrategy{ MaxUnavailable: maxUnavailable, }, diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go new file mode 100644 index 00000000..98570d90 --- /dev/null +++ b/controllers/provisioners/eks/rollingupdate.go @@ -0,0 +1,302 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eks + +import ( + "os" + "reflect" + "sort" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/go-logr/logr" + "github.com/keikoproj/instance-manager/controllers/common" + awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" + kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" + "github.com/keikoproj/instance-manager/controllers/provisioners/eks/scaling" + "github.com/pkg/errors" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + drain "k8s.io/kubectl/pkg/drain" +) + +const ( + RollingUpdateStrategyName = "rollingupdate" +) + +var ( + DefaultWaitGroupTimeout = time.Second * 3 + TransientLifecycleStates = []string{ + autoscaling.LifecycleStateDetaching, + autoscaling.LifecycleStateEnteringStandby, + autoscaling.LifecycleStatePending, + autoscaling.LifecycleStatePendingProceed, + autoscaling.LifecycleStatePendingWait, + autoscaling.LifecycleStateTerminating, + autoscaling.LifecycleStateTerminatingWait, + autoscaling.LifecycleStateTerminatingProceed, + autoscaling.LifecycleStateTerminated, + } +) + +type RollingUpdateRequest struct { + logr.Logger + AwsWorker awsprovider.AwsWorker + KubernetesClient kubeprovider.KubernetesClientSet + ClusterNodes *corev1.NodeList + DrainGroup *sync.WaitGroup + ScalingGroup *autoscaling.Group + MaxUnavailable int + DesiredCapacity int + AllInstances []string + UpdateTargets []string +} + +func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequest { + var ( + needsUpdate []string + allInstances []string + instanceGroup = ctx.GetInstanceGroup() + state = ctx.GetDiscoveredState() + scalingConfig = state.GetScalingConfiguration() + scalingResource = scalingConfig.Resource() + scalingGroup = state.GetScalingGroup() + desiredCount = int(aws.Int64Value(scalingGroup.DesiredCapacity)) + strategy = instanceGroup.GetUpgradeStrategy().GetRollingUpdateType() + maxUnavailable = strategy.GetMaxUnavailable() + ) + + // Get all Autoscaling Instances that needs update + for _, instance := range scalingGroup.Instances { + var ( + instanceId = aws.StringValue(instance.InstanceId) + lifecycle = aws.StringValue(instance.LifecycleState) + ) + + if common.ContainsEqualFold(TransientLifecycleStates, lifecycle) { + continue + } + + allInstances = append(allInstances, aws.StringValue(instance.InstanceId)) + + if awsprovider.IsUsingLaunchConfiguration(scalingGroup) { + if instance.LaunchConfigurationName == nil { + needsUpdate = append(needsUpdate, instanceId) + continue + } + + var ( + config = aws.StringValue(instance.LaunchConfigurationName) + activeConfig = aws.StringValue(scalingGroup.LaunchConfigurationName) + ) + + if !strings.EqualFold(config, activeConfig) { + needsUpdate = append(needsUpdate, instanceId) + } + } + + if awsprovider.IsUsingLaunchTemplate(scalingGroup) { + if instance.LaunchTemplate == nil { + needsUpdate = append(needsUpdate, instanceId) + continue + } + + var ( + config = aws.StringValue(instance.LaunchTemplate.LaunchTemplateName) + version = aws.StringValue(instance.LaunchTemplate.Version) + launchTemplate = scaling.ConvertToLaunchTemplate(scalingResource) + activeConfig = aws.StringValue(scalingGroup.LaunchTemplate.LaunchTemplateName) + activeVersionNum = aws.Int64Value(launchTemplate.LatestVersionNumber) + activeVersion = common.Int64ToStr(activeVersionNum) + ) + if !strings.EqualFold(config, activeConfig) || !strings.EqualFold(version, activeVersion) { + needsUpdate = append(needsUpdate, instanceId) + } + } + + if awsprovider.IsUsingMixedInstances(scalingGroup) { + if instance.LaunchTemplate == nil { + needsUpdate = append(needsUpdate, instanceId) + continue + } + + var ( + config = aws.StringValue(instance.LaunchTemplate.LaunchTemplateName) + version = aws.StringValue(instance.LaunchTemplate.Version) + launchTemplate = scaling.ConvertToLaunchTemplate(scalingResource) + activeConfig = aws.StringValue(scalingGroup.MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateName) + activeVersionNum = aws.Int64Value(launchTemplate.LatestVersionNumber) + activeVersion = common.Int64ToStr(activeVersionNum) + ) + + if !strings.EqualFold(config, activeConfig) || !strings.EqualFold(version, activeVersion) { + needsUpdate = append(needsUpdate, instanceId) + } + } + + } + allCount := len(allInstances) + + var unavailableInt int + if maxUnavailable.Type == intstr.String { + unavailableInt, _ = intstr.GetValueFromIntOrPercent(maxUnavailable, allCount, true) + } else { + unavailableInt = maxUnavailable.IntValue() + } + + if unavailableInt == 0 { + unavailableInt = 1 + } + + sort.Strings(needsUpdate) + + return &RollingUpdateRequest{ + Logger: ctx.Log, + KubernetesClient: ctx.KubernetesClient, + AwsWorker: ctx.AwsWorker, + DrainGroup: ctx.DrainGroup, + ClusterNodes: state.GetClusterNodes(), + MaxUnavailable: unavailableInt, + DesiredCapacity: desiredCount, + AllInstances: allInstances, + UpdateTargets: needsUpdate, + ScalingGroup: scalingGroup, + } +} + +func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { + + var ( + scalingGroupName = aws.StringValue(r.ScalingGroup.AutoScalingGroupName) + ) + + r.Info("starting rolling update", "scalinggroup", scalingGroupName, "targets", r.UpdateTargets, "maxunavailable", r.MaxUnavailable) + + if len(r.UpdateTargets) == 0 { + r.Info("no updatable instances", "scalinggroup", scalingGroupName) + return true, nil + } + + // cannot rotate if maxUnavailable is greater than number of desired + if r.MaxUnavailable > r.DesiredCapacity { + r.Info("maxUnavailable exceeds desired capacity, setting maxUnavailable match desired", + "scalinggroup", scalingGroupName, "maxunavailable", r.MaxUnavailable, "desiredcapacity", r.DesiredCapacity) + r.MaxUnavailable = r.DesiredCapacity + } + + ok := awsprovider.IsDesiredInService(r.ScalingGroup) + + if !ok { + r.Info("desired instances are not in service", "scalinggroup", scalingGroupName) + return false, nil + } + + ok, err := kubeprovider.IsMinNodesReady(r.ClusterNodes, r.AllInstances, r.MaxUnavailable) + if err != nil { + return false, err + } + + if !ok { + r.Info("desired nodes are not ready", "scalinggroup", scalingGroupName) + return false, nil + } + + var terminateTargets []string + if r.MaxUnavailable <= len(r.UpdateTargets) { + terminateTargets = r.UpdateTargets[:r.MaxUnavailable] + } else { + terminateTargets = r.UpdateTargets + } + + targetNodes := kubeprovider.GetNodesByInstance(terminateTargets, r.ClusterNodes) + + targetNames := make([]string, 0) + for _, node := range targetNodes.Items { + targetNames = append(targetNames, node.GetName()) + } + + r.Info("starting rotation on target nodes", "targets", targetNames) + + // Only create new threads if waitgroup is empty + drainErrs := make(chan error) + if !reflect.DeepEqual(r.DrainGroup, sync.WaitGroup{}) { + for _, node := range targetNodes.Items { + + nodeName := node.GetName() + r.Info("creating drainer goroutine for node", "node", nodeName) + + drainOpts := &drain.Helper{ + DeleteLocalData: true, + Force: true, + IgnoreAllDaemonSets: true, + GracePeriodSeconds: -1, + Client: r.KubernetesClient.Kubernetes, + Out: os.Stdout, + ErrOut: os.Stderr, + } + + r.DrainGroup.Add(1) + n := node + + go func() { + defer r.DrainGroup.Done() + r.Info("cordoning node", "node", nodeName) + err := drain.RunCordonOrUncordon(drainOpts, &n, true) + if err != nil { + errors.Wrap(err, "cordon node failed") + drainErrs <- err + } + r.Info("draining node", "node", nodeName) + err = drain.RunNodeDrain(drainOpts, nodeName) + if err != nil { + // If drain has failed, try to uncordon + drain.RunCordonOrUncordon(drainOpts, &n, false) + errors.Wrap(err, "drain node failed") + drainErrs <- err + } + }() + } + } + + timeout := make(chan struct{}) + go func() { + defer close(timeout) + r.DrainGroup.Wait() + }() + + select { + case err := <-drainErrs: + r.Info("failed to cordon/drain targets", "scalinggroup", scalingGroupName, "targets", terminateTargets) + return false, err + case <-timeout: + // goroutines completed, terminate and requeue + r.Info("targets drained successfully, terminating", "scalinggroup", scalingGroupName, "targets", terminateTargets) + if err := r.AwsWorker.TerminateScalingInstances(terminateTargets); err != nil { + // terminate failures are retryable + r.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", scalingGroupName, "targets", terminateTargets) + } + return false, nil + + case <-time.After(DefaultWaitGroupTimeout): + // goroutines timed out - requeue + r.Info("targets still draining", "scalinggroup", scalingGroupName, "targets", terminateTargets) + return false, nil + } +} diff --git a/controllers/provisioners/eks/upgrade.go b/controllers/provisioners/eks/upgrade.go index 9abb70fb..a03e163b 100644 --- a/controllers/provisioners/eks/upgrade.go +++ b/controllers/provisioners/eks/upgrade.go @@ -23,10 +23,8 @@ import ( "github.com/keikoproj/instance-manager/controllers/common" awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" - "github.com/keikoproj/instance-manager/controllers/provisioners/eks/scaling" "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/intstr" ) func (ctx *EksInstanceGroupContext) UpgradeNodes() error { @@ -52,11 +50,11 @@ func (ctx *EksInstanceGroupContext) UpgradeNodes() error { break } return nil - case kubeprovider.RollingUpdateStrategyName: + case RollingUpdateStrategyName: req := ctx.NewRollingUpdateRequest() - ok, err := kubeprovider.ProcessRollingUpgradeStrategy(req) + ok, err := req.ProcessRollingUpgradeStrategy() if err != nil { - state.Publisher.Publish(kubeprovider.InstanceGroupUpgradeFailedEvent, "instancegroup", instanceGroup.GetName(), "type", kubeprovider.RollingUpdateStrategyName, "error", err) + state.Publisher.Publish(kubeprovider.InstanceGroupUpgradeFailedEvent, "instancegroup", instanceGroup.GetName(), "type", RollingUpdateStrategyName, "error", err) instanceGroup.SetState(v1alpha1.ReconcileErr) return errors.Wrap(err, "failed to process rolling-update strategy") } @@ -92,106 +90,3 @@ func (ctx *EksInstanceGroupContext) BootstrapNodes() error { return common.UpsertAuthConfigMap(ctx.KubernetesClient.Kubernetes, []string{roleARN}, []string{osFamily}) } - -func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *kubeprovider.RollingUpdateRequest { - var ( - needsUpdate []string - allInstances []string - instanceGroup = ctx.GetInstanceGroup() - state = ctx.GetDiscoveredState() - scalingConfig = state.GetScalingConfiguration() - scalingResource = scalingConfig.Resource() - scalingGroup = state.GetScalingGroup() - desiredCount = int(aws.Int64Value(scalingGroup.DesiredCapacity)) - strategy = instanceGroup.GetUpgradeStrategy().GetRollingUpdateType() - maxUnavailable = strategy.GetMaxUnavailable() - asgName = aws.StringValue(scalingGroup.AutoScalingGroupName) - ) - - // Get all Autoscaling Instances that needs update - for _, instance := range scalingGroup.Instances { - var ( - instanceId = aws.StringValue(instance.InstanceId) - ) - - allInstances = append(allInstances, aws.StringValue(instance.InstanceId)) - - if awsprovider.IsUsingLaunchConfiguration(scalingGroup) { - if instance.LaunchConfigurationName == nil { - needsUpdate = append(needsUpdate, instanceId) - continue - } - - var ( - config = aws.StringValue(instance.LaunchConfigurationName) - activeConfig = aws.StringValue(scalingGroup.LaunchConfigurationName) - ) - - if !strings.EqualFold(config, activeConfig) { - needsUpdate = append(needsUpdate, instanceId) - } - } - - if awsprovider.IsUsingLaunchTemplate(scalingGroup) { - if instance.LaunchTemplate == nil { - needsUpdate = append(needsUpdate, instanceId) - continue - } - - var ( - config = aws.StringValue(instance.LaunchTemplate.LaunchTemplateName) - version = aws.StringValue(instance.LaunchTemplate.Version) - launchTemplate = scaling.ConvertToLaunchTemplate(scalingResource) - activeConfig = aws.StringValue(scalingGroup.LaunchTemplate.LaunchTemplateName) - activeVersionNum = aws.Int64Value(launchTemplate.LatestVersionNumber) - activeVersion = common.Int64ToStr(activeVersionNum) - ) - if !strings.EqualFold(config, activeConfig) || !strings.EqualFold(version, activeVersion) { - needsUpdate = append(needsUpdate, instanceId) - } - } - - if awsprovider.IsUsingMixedInstances(scalingGroup) { - if instance.LaunchTemplate == nil { - needsUpdate = append(needsUpdate, instanceId) - continue - } - - var ( - config = aws.StringValue(instance.LaunchTemplate.LaunchTemplateName) - version = aws.StringValue(instance.LaunchTemplate.Version) - launchTemplate = scaling.ConvertToLaunchTemplate(scalingResource) - activeConfig = aws.StringValue(scalingGroup.MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateName) - activeVersionNum = aws.Int64Value(launchTemplate.LatestVersionNumber) - activeVersion = common.Int64ToStr(activeVersionNum) - ) - - if !strings.EqualFold(config, activeConfig) || !strings.EqualFold(version, activeVersion) { - needsUpdate = append(needsUpdate, instanceId) - } - } - - } - allCount := len(allInstances) - - var unavailableInt int - if maxUnavailable.Type == intstr.String { - unavailableInt, _ = intstr.GetValueFromIntOrPercent(maxUnavailable, allCount, true) - } else { - unavailableInt = maxUnavailable.IntValue() - } - - if unavailableInt == 0 { - unavailableInt = 1 - } - - return &kubeprovider.RollingUpdateRequest{ - AwsWorker: ctx.AwsWorker, - ClusterNodes: state.GetClusterNodes(), - MaxUnavailable: unavailableInt, - DesiredCapacity: desiredCount, - AllInstances: allInstances, - UpdateTargets: needsUpdate, - ScalingGroupName: asgName, - } -} diff --git a/controllers/provisioners/provisioners.go b/controllers/provisioners/provisioners.go index fd953007..5bdcfba6 100644 --- a/controllers/provisioners/provisioners.go +++ b/controllers/provisioners/provisioners.go @@ -1,6 +1,8 @@ package provisioners import ( + "sync" + "github.com/go-logr/logr" "github.com/keikoproj/instance-manager/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -25,6 +27,7 @@ const ( ) type ProvisionerInput struct { + DrainGroup *sync.WaitGroup AwsWorker awsprovider.AwsWorker Kubernetes kubeprovider.KubernetesClientSet InstanceGroup *v1alpha1.InstanceGroup diff --git a/go.mod b/go.mod index 2380a914..24a26cab 100644 --- a/go.mod +++ b/go.mod @@ -15,9 +15,10 @@ require ( github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.6.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect + golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5 // indirect k8s.io/api v0.19.6 k8s.io/apimachinery v0.19.6 k8s.io/client-go v0.19.6 + k8s.io/kubectl v0.19.6 sigs.k8s.io/controller-runtime v0.7.0 - sigs.k8s.io/controller-tools v0.4.1 // indirect ) diff --git a/go.sum b/go.sum index b16b0a00..5172aa16 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= @@ -28,14 +29,18 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -61,6 +66,7 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -75,7 +81,6 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -85,25 +90,31 @@ github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAEl github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -140,11 +151,13 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -158,6 +171,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -167,13 +181,12 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/flect v0.2.0 h1:EWCvMGGxOjsgwlWaP+f4+Hh6yrrte7JeFL2S6b+0hdM= -github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -188,7 +201,6 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -202,7 +214,11 @@ github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= +github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -225,15 +241,13 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -264,7 +278,6 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -293,26 +306,29 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -338,12 +354,13 @@ github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -380,6 +397,7 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -426,12 +444,12 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -459,7 +477,6 @@ go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -468,7 +485,6 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -495,7 +511,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -515,7 +530,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -536,16 +550,13 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -558,7 +569,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -572,7 +582,6 @@ golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -606,7 +615,6 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -651,7 +659,6 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -692,9 +699,9 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -703,60 +710,51 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.6 h1:F3lfwgpKcKms6F1mMqkQXFzXmme8QqHTJBtBkev3TOg= k8s.io/api v0.19.6/go.mod h1:Plxx44Nh4zVblkJrIgxVPgPre1mvng6tXf1Sj3bs0fU= -k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= -k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.6 h1:kBLzSGuDdY1NdSV2uFzI+FwZ9wtkmG+X3ZVcWXSqNgA= k8s.io/apimachinery v0.19.6/go.mod h1:6sRbGRAVY5DOCuZwB5XkqguBqpqLU6q/kOaOdk29z6Q= -k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= -k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= +k8s.io/cli-runtime v0.19.6 h1:PwRKKVbCFzcGJ9WxctV6liI95GMiAneJrSFZTr6Za94= +k8s.io/cli-runtime v0.19.6/go.mod h1:VOnivMtWLPKVuFBEVS9jM1WbR9TynCIja9H5jwviY/c= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/client-go v0.19.6 h1:vtPb33nP8DBMW+/CyuJ8fiie36c3CM1Ts6L4Tsr+PtU= k8s.io/client-go v0.19.6/go.mod h1:gEiS+efRlXYUEQ9Oz4lmNXlxAl5JZ8y2zbTDGhvXXnk= -k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= +k8s.io/code-generator v0.19.6/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/component-base v0.19.2 h1:jW5Y9RcZTb79liEhW3XDVTW7MuvEGP0tQZnfSX6/+gs= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/component-base v0.19.6 h1:V76d3rIEWvP95peWgRycKslQnEwlaPy4UORvh3+YBbU= +k8s.io/component-base v0.19.6/go.mod h1:8Btsf8J00/fVDa/YFmXjei7gVkcFrlKZXjSeP4SZNJg= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/kubectl v0.19.6 h1:7FpynLRAyCuvVQtoV/bwwNKcwZTnprVHc6fui57KiRc= +k8s.io/kubectl v0.19.6/go.mod h1:SenrDhlFxxBHnf4amqNPY2ajWJ3CDMhrcm3sCOY4pLI= +k8s.io/metrics v0.19.6/go.mod h1:jM61saf/bjMRmow6zan2cAk8vFDmqvbNXFRbB4g7TNs= k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= -sigs.k8s.io/controller-tools v0.4.1 h1:VkuV0MxlRPmRu5iTgBZU4UxUX2LiR99n3sdQGRxZF4w= -sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/main.go b/main.go index 4f66251a..0779aceb 100644 --- a/main.go +++ b/main.go @@ -150,6 +150,7 @@ func main() { Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("instancegroup"), MaxParallel: maxParallel, + DrainGroups: &sync.Map{}, Auth: &controllers.InstanceGroupAuthenticator{ Aws: awsWorker, Kubernetes: kube, From 5885f75a5bdc11af506a3c121bc3687cf22e2d25 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Tue, 16 Feb 2021 17:28:09 -0800 Subject: [PATCH 02/11] fixes Signed-off-by: Eytan Avisror --- controllers/provisioners/eks/rollingupdate.go | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index 98570d90..afe7a0bc 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -187,22 +187,7 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { scalingGroupName = aws.StringValue(r.ScalingGroup.AutoScalingGroupName) ) - r.Info("starting rolling update", "scalinggroup", scalingGroupName, "targets", r.UpdateTargets, "maxunavailable", r.MaxUnavailable) - - if len(r.UpdateTargets) == 0 { - r.Info("no updatable instances", "scalinggroup", scalingGroupName) - return true, nil - } - - // cannot rotate if maxUnavailable is greater than number of desired - if r.MaxUnavailable > r.DesiredCapacity { - r.Info("maxUnavailable exceeds desired capacity, setting maxUnavailable match desired", - "scalinggroup", scalingGroupName, "maxunavailable", r.MaxUnavailable, "desiredcapacity", r.DesiredCapacity) - r.MaxUnavailable = r.DesiredCapacity - } - ok := awsprovider.IsDesiredInService(r.ScalingGroup) - if !ok { r.Info("desired instances are not in service", "scalinggroup", scalingGroupName) return false, nil @@ -218,6 +203,20 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { return false, nil } + r.Info("starting rolling update", "scalinggroup", scalingGroupName, "targets", r.UpdateTargets, "maxunavailable", r.MaxUnavailable) + + if len(r.UpdateTargets) == 0 { + r.Info("no updatable instances", "scalinggroup", scalingGroupName) + return true, nil + } + + // cannot rotate if maxUnavailable is greater than number of desired + if r.MaxUnavailable > r.DesiredCapacity { + r.Info("maxUnavailable exceeds desired capacity, setting maxUnavailable match desired", + "scalinggroup", scalingGroupName, "maxunavailable", r.MaxUnavailable, "desiredcapacity", r.DesiredCapacity) + r.MaxUnavailable = r.DesiredCapacity + } + var terminateTargets []string if r.MaxUnavailable <= len(r.UpdateTargets) { terminateTargets = r.UpdateTargets[:r.MaxUnavailable] @@ -236,7 +235,7 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { // Only create new threads if waitgroup is empty drainErrs := make(chan error) - if !reflect.DeepEqual(r.DrainGroup, sync.WaitGroup{}) { + if !reflect.DeepEqual(r.DrainGroup, &sync.WaitGroup{}) { for _, node := range targetNodes.Items { nodeName := node.GetName() From 4fe1855049bb01b00fb3e79ec6b0143516a9d64e Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Tue, 16 Feb 2021 17:34:35 -0800 Subject: [PATCH 03/11] Update rollingupdate.go Signed-off-by: Eytan Avisror --- controllers/provisioners/eks/rollingupdate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index afe7a0bc..dba2381e 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -235,7 +235,7 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { // Only create new threads if waitgroup is empty drainErrs := make(chan error) - if !reflect.DeepEqual(r.DrainGroup, &sync.WaitGroup{}) { + if reflect.DeepEqual(r.DrainGroup, &sync.WaitGroup{}) { for _, node := range targetNodes.Items { nodeName := node.GetName() From 5187ca8e180f2d898f55d1947ca365800f256045 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Wed, 17 Feb 2021 15:40:24 -0800 Subject: [PATCH 04/11] Fixes and options Signed-off-by: Eytan Avisror --- api/v1alpha1/instancegroup_types.go | 87 ++++++++++++++++-- api/v1alpha1/zz_generated.deepcopy.go | 46 ++++++++++ ...stancemgr.keikoproj.io_instancegroups.yaml | 20 +++++ controllers/instancegroup_controller.go | 12 ++- controllers/providers/kubernetes/utils.go | 6 ++ controllers/provisioners/eks/eks.go | 4 +- controllers/provisioners/eks/rollingupdate.go | 88 +++++++++++++------ controllers/provisioners/eks/upgrade.go | 2 +- controllers/provisioners/provisioners.go | 4 +- main.go | 3 +- 10 files changed, 230 insertions(+), 42 deletions(-) diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go index 8acb5522..f900bdd2 100644 --- a/api/v1alpha1/instancegroup_types.go +++ b/api/v1alpha1/instancegroup_types.go @@ -97,12 +97,22 @@ var ( LaunchTemplate, } - DefaultRollingUpdateStrategy = &RollingUpdateStrategy{ + DefaultRollingUpgradeTimeoutSeconds int64 = 900 + DefaultRollingUpgradeForce = true + DefaultRollingUpdateStrategy = &RollingUpdateStrategy{ MaxUnavailable: &intstr.IntOrString{ Type: intstr.Int, IntVal: 1, }, + DrainOptions: RollingUpgradeDrainOptions{ + TimeoutSeconds: &DefaultRollingUpgradeTimeoutSeconds, + Force: &DefaultRollingUpgradeForce, + }, } + + NamespaceReadinessGateType = "namespace" + AllowedReadinessGateTypes = []string{NamespaceReadinessGateType} + DefaultCRDStrategyMaxRetries = 3 AllowedFileSystemTypes = []string{FileSystemTypeXFS, FileSystemTypeEXT4} @@ -150,7 +160,17 @@ type AwsUpgradeStrategy struct { } type RollingUpdateStrategy struct { - MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + DrainOptions RollingUpgradeDrainOptions `json:"drainOptions,omitempty"` + ReadinessGates []RollingUpgradeReadinessGate `json:"readinessGates,omitempty"` +} + +func (s *RollingUpdateStrategy) GetReadinessGates() []RollingUpgradeReadinessGate { + return s.ReadinessGates +} + +func (s *RollingUpdateStrategy) GetDrainOptions() RollingUpgradeDrainOptions { + return s.DrainOptions } func (s *RollingUpdateStrategy) GetMaxUnavailable() *intstr.IntOrString { @@ -161,6 +181,32 @@ func (s *RollingUpdateStrategy) SetMaxUnavailable(value *intstr.IntOrString) { s.MaxUnavailable = value } +type RollingUpgradeReadinessGate struct { + Type string `json:"type"` + Value string `json:"value"` +} + +type RollingUpgradeDrainOptions struct { + TimeoutSeconds *int64 `json:"timeoutSeconds,omitempty"` + Force *bool `json:"force,omitempty"` +} + +func (d *RollingUpgradeDrainOptions) GetTimeoutSeconds() int64 { + return *d.TimeoutSeconds +} + +func (d *RollingUpgradeDrainOptions) SetTimeoutSeconds(n *int64) { + d.TimeoutSeconds = n +} + +func (d *RollingUpgradeDrainOptions) GetForce() bool { + return *d.Force +} + +func (d *RollingUpgradeDrainOptions) SetForce(condition *bool) { + d.Force = condition +} + type CRDUpdateStrategy struct { Spec string `json:"spec,omitempty"` CRDName string `json:"crdName,omitempty"` @@ -656,12 +702,14 @@ func (ig *InstanceGroup) Validate() error { } } - if strings.EqualFold(s.AwsUpgradeStrategy.Type, RollingUpdateStrategyName) && s.AwsUpgradeStrategy.RollingUpdateType == nil { - s.AwsUpgradeStrategy.RollingUpdateType = DefaultRollingUpdateStrategy + if strings.EqualFold(s.AwsUpgradeStrategy.Type, RollingUpdateStrategyName) { + if err := s.AwsUpgradeStrategy.RollingUpdateType.Validate(); err != nil { + return err + } } - return nil } + func (c *EKSConfiguration) GetRoleName() string { return c.ExistingRoleName } @@ -834,6 +882,35 @@ func (s *AwsUpgradeStrategy) SetCRDType(crd *CRDUpdateStrategy) { s.CRDType = crd } +func (r *RollingUpdateStrategy) Validate() error { + if r == nil { + r = DefaultRollingUpdateStrategy + } + + if r.MaxUnavailable == nil { + r.MaxUnavailable = DefaultRollingUpdateStrategy.MaxUnavailable + } + + if r.DrainOptions.TimeoutSeconds == nil { + r.DrainOptions.TimeoutSeconds = DefaultRollingUpdateStrategy.DrainOptions.TimeoutSeconds + } + + if r.DrainOptions.Force == nil { + r.DrainOptions.Force = DefaultRollingUpdateStrategy.DrainOptions.Force + } + + for _, gate := range r.ReadinessGates { + if !common.ContainsEqualFold(AllowedReadinessGateTypes, gate.Type) { + return errors.Errorf("readiness gate type '%v' is unsupported, supported types are '%v'", gate.Type, AllowedReadinessGateTypes) + } + if common.StringEmpty(gate.Value) { + return errors.New("readiness gate value cannot be empty") + } + } + + return nil +} + func (c *CRDUpdateStrategy) Validate() error { if c.GetSpec() == "" { return errors.New("spec is empty") diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 02afd239..a2618fdb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -615,6 +615,12 @@ func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { *out = new(intstr.IntOrString) **out = **in } + in.DrainOptions.DeepCopyInto(&out.DrainOptions) + if in.ReadinessGates != nil { + in, out := &in.ReadinessGates, &out.ReadinessGates + *out = make([]RollingUpgradeReadinessGate, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStrategy. @@ -627,6 +633,46 @@ func (in *RollingUpdateStrategy) DeepCopy() *RollingUpdateStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpgradeDrainOptions) DeepCopyInto(out *RollingUpgradeDrainOptions) { + *out = *in + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int64) + **out = **in + } + if in.Force != nil { + in, out := &in.Force, &out.Force + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpgradeDrainOptions. +func (in *RollingUpgradeDrainOptions) DeepCopy() *RollingUpgradeDrainOptions { + if in == nil { + return nil + } + out := new(RollingUpgradeDrainOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpgradeReadinessGate) DeepCopyInto(out *RollingUpgradeReadinessGate) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpgradeReadinessGate. +func (in *RollingUpgradeReadinessGate) DeepCopy() *RollingUpgradeReadinessGate { + if in == nil { + return nil + } + out := new(RollingUpgradeReadinessGate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserDataStage) DeepCopyInto(out *UserDataStage) { *out = *in diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml index 32347cfa..798b2f28 100644 --- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml +++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml @@ -379,11 +379,31 @@ spec: type: object rollingUpdate: properties: + drainOptions: + properties: + force: + type: boolean + timeoutSeconds: + format: int64 + type: integer + type: object maxUnavailable: anyOf: - type: integer - type: string x-kubernetes-int-or-string: true + readinessGates: + items: + properties: + type: + type: string + value: + type: string + required: + - type + - value + type: object + type: array type: object type: type: string diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go index b3006f7e..d9851be4 100644 --- a/controllers/instancegroup_controller.go +++ b/controllers/instancegroup_controller.go @@ -52,7 +52,9 @@ type InstanceGroupReconciler struct { ConfigMap *corev1.ConfigMap Namespaces map[string]corev1.Namespace NamespacesLock *sync.Mutex - DrainGroups *sync.Map + DrainManager kubeprovider.DrainManager + DrainGroupMapper *sync.Map + DrainErrorMapper *sync.Map ConfigRetention int } @@ -141,8 +143,12 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque ) status.SetConfigHash(configHash) - drainGroup, _ := r.DrainGroups.LoadOrStore(namespacedName, &sync.WaitGroup{}) - input.DrainGroup = drainGroup.(*sync.WaitGroup) + drainGroup, _ := r.DrainGroupMapper.LoadOrStore(namespacedName, &sync.WaitGroup{}) + drainErrs, _ := r.DrainErrorMapper.LoadOrStore(namespacedName, make(chan error)) + input.DrainManager = kubeprovider.DrainManager{ + DrainErrors: drainErrs.(chan error), + DrainGroup: drainGroup.(*sync.WaitGroup), + } if !reflect.DeepEqual(*r.ConfigMap, corev1.ConfigMap{}) { // Configmap exist - apply defaults/boundaries if namespace is not excluded diff --git a/controllers/providers/kubernetes/utils.go b/controllers/providers/kubernetes/utils.go index c21c3428..4d6e3e97 100644 --- a/controllers/providers/kubernetes/utils.go +++ b/controllers/providers/kubernetes/utils.go @@ -25,6 +25,7 @@ import ( "os/user" "reflect" "strings" + "sync" jsonpatch "github.com/evanphx/json-patch" "github.com/ghodss/yaml" @@ -53,6 +54,11 @@ type KubernetesClientSet struct { KubeDynamic dynamic.Interface } +type DrainManager struct { + DrainErrors chan error + DrainGroup *sync.WaitGroup +} + func GetUnstructuredInstanceGroup(instanceGroup *v1alpha1.InstanceGroup) (*unstructured.Unstructured, error) { var obj = &unstructured.Unstructured{} content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(instanceGroup) diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go index 7d0cef0c..28020d43 100644 --- a/controllers/provisioners/eks/eks.go +++ b/controllers/provisioners/eks/eks.go @@ -63,7 +63,7 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext { ctx := &EksInstanceGroupContext{ InstanceGroup: instanceGroup, - DrainGroup: p.DrainGroup, + DrainManager: p.DrainManager, KubernetesClient: p.Kubernetes, AwsWorker: p.AwsWorker, Log: p.Log.WithName("eks"), @@ -80,7 +80,7 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext { type EksInstanceGroupContext struct { sync.Mutex - DrainGroup *sync.WaitGroup + DrainManager kubeprovider.DrainManager InstanceGroup *v1alpha1.InstanceGroup KubernetesClient kubeprovider.KubernetesClientSet AwsWorker awsprovider.AwsWorker diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index dba2381e..25149c1a 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -16,7 +16,8 @@ limitations under the License. package eks import ( - "os" + "bytes" + "fmt" "reflect" "sort" "strings" @@ -26,6 +27,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/go-logr/logr" + "github.com/keikoproj/instance-manager/api/v1alpha1" "github.com/keikoproj/instance-manager/controllers/common" awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" @@ -42,7 +44,7 @@ const ( ) var ( - DefaultWaitGroupTimeout = time.Second * 3 + DefaultWaitGroupTimeout = time.Second * 5 TransientLifecycleStates = []string{ autoscaling.LifecycleStateDetaching, autoscaling.LifecycleStateEnteringStandby, @@ -61,12 +63,16 @@ type RollingUpdateRequest struct { AwsWorker awsprovider.AwsWorker KubernetesClient kubeprovider.KubernetesClientSet ClusterNodes *corev1.NodeList - DrainGroup *sync.WaitGroup - ScalingGroup *autoscaling.Group - MaxUnavailable int - DesiredCapacity int - AllInstances []string - UpdateTargets []string + // DrainGroup *sync.WaitGroup + ScalingGroup *autoscaling.Group + DrainOptions v1alpha1.RollingUpgradeDrainOptions + // DrainErrors chan error + DrainManager kubeprovider.DrainManager + ReadinessGates []v1alpha1.RollingUpgradeReadinessGate + MaxUnavailable int + DesiredCapacity int + AllInstances []string + UpdateTargets []string } func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequest { @@ -81,6 +87,8 @@ func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequ desiredCount = int(aws.Int64Value(scalingGroup.DesiredCapacity)) strategy = instanceGroup.GetUpgradeStrategy().GetRollingUpdateType() maxUnavailable = strategy.GetMaxUnavailable() + drainOpts = strategy.GetDrainOptions() + readinessGates = strategy.GetReadinessGates() ) // Get all Autoscaling Instances that needs update @@ -171,7 +179,9 @@ func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequ Logger: ctx.Log, KubernetesClient: ctx.KubernetesClient, AwsWorker: ctx.AwsWorker, - DrainGroup: ctx.DrainGroup, + DrainManager: ctx.DrainManager, + DrainOptions: drainOpts, + ReadinessGates: readinessGates, ClusterNodes: state.GetClusterNodes(), MaxUnavailable: unavailableInt, DesiredCapacity: desiredCount, @@ -185,6 +195,8 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { var ( scalingGroupName = aws.StringValue(r.ScalingGroup.AutoScalingGroupName) + drainConfig = r.DrainOptions + instanceCount = len(r.AllInstances) ) ok := awsprovider.IsDesiredInService(r.ScalingGroup) @@ -193,11 +205,10 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { return false, nil } - ok, err := kubeprovider.IsMinNodesReady(r.ClusterNodes, r.AllInstances, r.MaxUnavailable) + ok, err := kubeprovider.IsMinNodesReady(r.ClusterNodes, r.AllInstances, instanceCount) if err != nil { return false, err } - if !ok { r.Info("desired nodes are not ready", "scalinggroup", scalingGroupName) return false, nil @@ -205,6 +216,15 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { r.Info("starting rolling update", "scalinggroup", scalingGroupName, "targets", r.UpdateTargets, "maxunavailable", r.MaxUnavailable) + ok, err = r.IsReadinessGateAllowed() + if err != nil { + return false, err + } + if !ok { + r.Info("readiness gates are not passing", "scalinggroup", scalingGroupName) + return false, nil + } + if len(r.UpdateTargets) == 0 { r.Info("no updatable instances", "scalinggroup", scalingGroupName) return true, nil @@ -234,41 +254,42 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { r.Info("starting rotation on target nodes", "targets", targetNames) // Only create new threads if waitgroup is empty - drainErrs := make(chan error) - if reflect.DeepEqual(r.DrainGroup, &sync.WaitGroup{}) { + if reflect.DeepEqual(r.DrainManager.DrainGroup, &sync.WaitGroup{}) { for _, node := range targetNodes.Items { - + buff := bytes.NewBufferString("") nodeName := node.GetName() r.Info("creating drainer goroutine for node", "node", nodeName) - + timeoutSeconds := time.Duration(drainConfig.GetTimeoutSeconds()) * time.Second + fmt.Println(timeoutSeconds) drainOpts := &drain.Helper{ DeleteLocalData: true, - Force: true, + Force: drainConfig.GetForce(), IgnoreAllDaemonSets: true, + Timeout: timeoutSeconds, GracePeriodSeconds: -1, Client: r.KubernetesClient.Kubernetes, - Out: os.Stdout, - ErrOut: os.Stderr, + Out: buff, + ErrOut: buff, } - r.DrainGroup.Add(1) + r.DrainManager.DrainGroup.Add(1) n := node go func() { - defer r.DrainGroup.Done() + defer r.DrainManager.DrainGroup.Done() r.Info("cordoning node", "node", nodeName) err := drain.RunCordonOrUncordon(drainOpts, &n, true) if err != nil { - errors.Wrap(err, "cordon node failed") - drainErrs <- err + werr := errors.Errorf("cordon node failed: %v, %v", err.Error(), buff.String()) + r.DrainManager.DrainErrors <- werr } r.Info("draining node", "node", nodeName) err = drain.RunNodeDrain(drainOpts, nodeName) if err != nil { // If drain has failed, try to uncordon drain.RunCordonOrUncordon(drainOpts, &n, false) - errors.Wrap(err, "drain node failed") - drainErrs <- err + werr := errors.Errorf("drain node failed: %v, %v", err.Error(), buff.String()) + r.DrainManager.DrainErrors <- werr } }() } @@ -277,13 +298,14 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { timeout := make(chan struct{}) go func() { defer close(timeout) - r.DrainGroup.Wait() + r.DrainManager.DrainGroup.Wait() }() select { - case err := <-drainErrs: - r.Info("failed to cordon/drain targets", "scalinggroup", scalingGroupName, "targets", terminateTargets) + case err := <-r.DrainManager.DrainErrors: + r.Info("failed to cordon/drain targets", "error", err, "scalinggroup", scalingGroupName, "targets", terminateTargets) return false, err + case <-timeout: // goroutines completed, terminate and requeue r.Info("targets drained successfully, terminating", "scalinggroup", scalingGroupName, "targets", terminateTargets) @@ -299,3 +321,15 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { return false, nil } } + +func (r *RollingUpdateRequest) IsReadinessGateAllowed() (bool, error) { + var ( + readinessGates = r.ReadinessGates + ) + + for _, gate := range readinessGates { + _ = gate + } + + return true, nil +} diff --git a/controllers/provisioners/eks/upgrade.go b/controllers/provisioners/eks/upgrade.go index a03e163b..28331e89 100644 --- a/controllers/provisioners/eks/upgrade.go +++ b/controllers/provisioners/eks/upgrade.go @@ -54,7 +54,7 @@ func (ctx *EksInstanceGroupContext) UpgradeNodes() error { req := ctx.NewRollingUpdateRequest() ok, err := req.ProcessRollingUpgradeStrategy() if err != nil { - state.Publisher.Publish(kubeprovider.InstanceGroupUpgradeFailedEvent, "instancegroup", instanceGroup.GetName(), "type", RollingUpdateStrategyName, "error", err) + state.Publisher.Publish(kubeprovider.InstanceGroupUpgradeFailedEvent, "instancegroup", instanceGroup.GetName(), "type", RollingUpdateStrategyName, "error", err.Error()) instanceGroup.SetState(v1alpha1.ReconcileErr) return errors.Wrap(err, "failed to process rolling-update strategy") } diff --git a/controllers/provisioners/provisioners.go b/controllers/provisioners/provisioners.go index 5bdcfba6..6c470f9b 100644 --- a/controllers/provisioners/provisioners.go +++ b/controllers/provisioners/provisioners.go @@ -1,8 +1,6 @@ package provisioners import ( - "sync" - "github.com/go-logr/logr" "github.com/keikoproj/instance-manager/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -27,7 +25,7 @@ const ( ) type ProvisionerInput struct { - DrainGroup *sync.WaitGroup + DrainManager kubeprovider.DrainManager AwsWorker awsprovider.AwsWorker Kubernetes kubeprovider.KubernetesClientSet InstanceGroup *v1alpha1.InstanceGroup diff --git a/main.go b/main.go index 0779aceb..2804580a 100644 --- a/main.go +++ b/main.go @@ -150,7 +150,8 @@ func main() { Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("instancegroup"), MaxParallel: maxParallel, - DrainGroups: &sync.Map{}, + DrainGroupMapper: &sync.Map{}, + DrainErrorMapper: &sync.Map{}, Auth: &controllers.InstanceGroupAuthenticator{ Aws: awsWorker, Kubernetes: kube, From 4fcf804fc1d7443f6eccc0a10bafb5bcf48b98fb Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Wed, 17 Feb 2021 15:44:01 -0800 Subject: [PATCH 05/11] Update instancegroup_controller.go Signed-off-by: Eytan Avisror --- controllers/instancegroup_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go index d9851be4..2f5f7860 100644 --- a/controllers/instancegroup_controller.go +++ b/controllers/instancegroup_controller.go @@ -52,7 +52,6 @@ type InstanceGroupReconciler struct { ConfigMap *corev1.ConfigMap Namespaces map[string]corev1.Namespace NamespacesLock *sync.Mutex - DrainManager kubeprovider.DrainManager DrainGroupMapper *sync.Map DrainErrorMapper *sync.Map ConfigRetention int From a02d38c6999c86f0946af0421e7e733aafd9b87b Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Wed, 17 Feb 2021 15:57:21 -0800 Subject: [PATCH 06/11] add max retries Signed-off-by: Eytan Avisror --- api/v1alpha1/instancegroup_types.go | 8 +++- api/v1alpha1/zz_generated.deepcopy.go | 5 +++ ...stancemgr.keikoproj.io_instancegroups.yaml | 2 + controllers/provisioners/eks/rollingupdate.go | 40 +++++++++++-------- 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go index f900bdd2..ac957e0b 100644 --- a/api/v1alpha1/instancegroup_types.go +++ b/api/v1alpha1/instancegroup_types.go @@ -113,7 +113,8 @@ var ( NamespaceReadinessGateType = "namespace" AllowedReadinessGateTypes = []string{NamespaceReadinessGateType} - DefaultCRDStrategyMaxRetries = 3 + DefaultRollingUpgradeStrategyMaxRetries = 3 + DefaultCRDStrategyMaxRetries = 3 AllowedFileSystemTypes = []string{FileSystemTypeXFS, FileSystemTypeEXT4} AllowedMixedPolicyStrategies = []string{LaunchTemplateStrategyCapacityOptimized, LaunchTemplateStrategyLowestPrice} @@ -161,6 +162,7 @@ type AwsUpgradeStrategy struct { type RollingUpdateStrategy struct { MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + MaxRetries *int `json:"maxRetries,omitempty"` DrainOptions RollingUpgradeDrainOptions `json:"drainOptions,omitempty"` ReadinessGates []RollingUpgradeReadinessGate `json:"readinessGates,omitempty"` } @@ -891,6 +893,10 @@ func (r *RollingUpdateStrategy) Validate() error { r.MaxUnavailable = DefaultRollingUpdateStrategy.MaxUnavailable } + if r.MaxRetries == nil { + r.MaxRetries = &DefaultRollingUpgradeStrategyMaxRetries + } + if r.DrainOptions.TimeoutSeconds == nil { r.DrainOptions.TimeoutSeconds = DefaultRollingUpdateStrategy.DrainOptions.TimeoutSeconds } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a2618fdb..6ec0bf50 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -615,6 +615,11 @@ func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { *out = new(intstr.IntOrString) **out = **in } + if in.MaxRetries != nil { + in, out := &in.MaxRetries, &out.MaxRetries + *out = new(int) + **out = **in + } in.DrainOptions.DeepCopyInto(&out.DrainOptions) if in.ReadinessGates != nil { in, out := &in.ReadinessGates, &out.ReadinessGates diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml index 798b2f28..f0e7506e 100644 --- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml +++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml @@ -387,6 +387,8 @@ spec: format: int64 type: integer type: object + maxRetries: + type: integer maxUnavailable: anyOf: - type: integer diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index 25149c1a..7b5b5832 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -60,19 +60,16 @@ var ( type RollingUpdateRequest struct { logr.Logger + InstanceGroup *v1alpha1.InstanceGroup AwsWorker awsprovider.AwsWorker KubernetesClient kubeprovider.KubernetesClientSet ClusterNodes *corev1.NodeList - // DrainGroup *sync.WaitGroup - ScalingGroup *autoscaling.Group - DrainOptions v1alpha1.RollingUpgradeDrainOptions - // DrainErrors chan error - DrainManager kubeprovider.DrainManager - ReadinessGates []v1alpha1.RollingUpgradeReadinessGate - MaxUnavailable int - DesiredCapacity int - AllInstances []string - UpdateTargets []string + ScalingGroup *autoscaling.Group + DrainManager kubeprovider.DrainManager + MaxUnavailable int + DesiredCapacity int + AllInstances []string + UpdateTargets []string } func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequest { @@ -87,8 +84,6 @@ func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequ desiredCount = int(aws.Int64Value(scalingGroup.DesiredCapacity)) strategy = instanceGroup.GetUpgradeStrategy().GetRollingUpdateType() maxUnavailable = strategy.GetMaxUnavailable() - drainOpts = strategy.GetDrainOptions() - readinessGates = strategy.GetReadinessGates() ) // Get all Autoscaling Instances that needs update @@ -177,11 +172,10 @@ func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequ return &RollingUpdateRequest{ Logger: ctx.Log, + InstanceGroup: instanceGroup, KubernetesClient: ctx.KubernetesClient, AwsWorker: ctx.AwsWorker, DrainManager: ctx.DrainManager, - DrainOptions: drainOpts, - ReadinessGates: readinessGates, ClusterNodes: state.GetClusterNodes(), MaxUnavailable: unavailableInt, DesiredCapacity: desiredCount, @@ -195,8 +189,10 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { var ( scalingGroupName = aws.StringValue(r.ScalingGroup.AutoScalingGroupName) - drainConfig = r.DrainOptions instanceCount = len(r.AllInstances) + strategy = r.InstanceGroup.GetUpgradeStrategy().GetRollingUpdateType() + drainConfig = strategy.GetDrainOptions() + status = r.InstanceGroup.GetStatus() ) ok := awsprovider.IsDesiredInService(r.ScalingGroup) @@ -304,6 +300,16 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { select { case err := <-r.DrainManager.DrainErrors: r.Info("failed to cordon/drain targets", "error", err, "scalinggroup", scalingGroupName, "targets", terminateTargets) + maxRetries := *strategy.MaxRetries + if maxRetries > status.GetStrategyRetryCount() { + if maxRetries == -1 { + // if maxRetries is set to -1, retry forever + status.SetStrategyRetryCount(-1) + } else { + // otherwise increment retry counter + status.IncrementStrategyRetryCount() + } + } return false, err case <-timeout: @@ -313,6 +319,7 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { // terminate failures are retryable r.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", scalingGroupName, "targets", terminateTargets) } + status.SetStrategyRetryCount(0) return false, nil case <-time.After(DefaultWaitGroupTimeout): @@ -324,7 +331,8 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { func (r *RollingUpdateRequest) IsReadinessGateAllowed() (bool, error) { var ( - readinessGates = r.ReadinessGates + strategy = r.InstanceGroup.GetUpgradeStrategy().GetRollingUpdateType() + readinessGates = strategy.GetReadinessGates() ) for _, gate := range readinessGates { From 11dcec92a87025488f6d7e7106bd447668008cd0 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Wed, 17 Feb 2021 20:28:26 -0800 Subject: [PATCH 07/11] remove max retries Signed-off-by: Eytan Avisror --- api/v1alpha1/instancegroup_types.go | 8 +------- api/v1alpha1/zz_generated.deepcopy.go | 5 ----- ...instancemgr.keikoproj.io_instancegroups.yaml | 2 -- controllers/provisioners/eks/rollingupdate.go | 17 +++-------------- controllers/provisioners/eks/update.go | 1 - 5 files changed, 4 insertions(+), 29 deletions(-) diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go index ac957e0b..f900bdd2 100644 --- a/api/v1alpha1/instancegroup_types.go +++ b/api/v1alpha1/instancegroup_types.go @@ -113,8 +113,7 @@ var ( NamespaceReadinessGateType = "namespace" AllowedReadinessGateTypes = []string{NamespaceReadinessGateType} - DefaultRollingUpgradeStrategyMaxRetries = 3 - DefaultCRDStrategyMaxRetries = 3 + DefaultCRDStrategyMaxRetries = 3 AllowedFileSystemTypes = []string{FileSystemTypeXFS, FileSystemTypeEXT4} AllowedMixedPolicyStrategies = []string{LaunchTemplateStrategyCapacityOptimized, LaunchTemplateStrategyLowestPrice} @@ -162,7 +161,6 @@ type AwsUpgradeStrategy struct { type RollingUpdateStrategy struct { MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` - MaxRetries *int `json:"maxRetries,omitempty"` DrainOptions RollingUpgradeDrainOptions `json:"drainOptions,omitempty"` ReadinessGates []RollingUpgradeReadinessGate `json:"readinessGates,omitempty"` } @@ -893,10 +891,6 @@ func (r *RollingUpdateStrategy) Validate() error { r.MaxUnavailable = DefaultRollingUpdateStrategy.MaxUnavailable } - if r.MaxRetries == nil { - r.MaxRetries = &DefaultRollingUpgradeStrategyMaxRetries - } - if r.DrainOptions.TimeoutSeconds == nil { r.DrainOptions.TimeoutSeconds = DefaultRollingUpdateStrategy.DrainOptions.TimeoutSeconds } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6ec0bf50..a2618fdb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -615,11 +615,6 @@ func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { *out = new(intstr.IntOrString) **out = **in } - if in.MaxRetries != nil { - in, out := &in.MaxRetries, &out.MaxRetries - *out = new(int) - **out = **in - } in.DrainOptions.DeepCopyInto(&out.DrainOptions) if in.ReadinessGates != nil { in, out := &in.ReadinessGates, &out.ReadinessGates diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml index f0e7506e..798b2f28 100644 --- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml +++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml @@ -387,8 +387,6 @@ spec: format: int64 type: integer type: object - maxRetries: - type: integer maxUnavailable: anyOf: - type: integer diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index 7b5b5832..bf8f5ab7 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -17,7 +17,6 @@ package eks import ( "bytes" - "fmt" "reflect" "sort" "strings" @@ -190,6 +189,7 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { var ( scalingGroupName = aws.StringValue(r.ScalingGroup.AutoScalingGroupName) instanceCount = len(r.AllInstances) + namespacedName = r.InstanceGroup.NamespacedName() strategy = r.InstanceGroup.GetUpgradeStrategy().GetRollingUpdateType() drainConfig = strategy.GetDrainOptions() status = r.InstanceGroup.GetStatus() @@ -252,11 +252,10 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { // Only create new threads if waitgroup is empty if reflect.DeepEqual(r.DrainManager.DrainGroup, &sync.WaitGroup{}) { for _, node := range targetNodes.Items { - buff := bytes.NewBufferString("") nodeName := node.GetName() r.Info("creating drainer goroutine for node", "node", nodeName) + buff := bytes.NewBufferString("") timeoutSeconds := time.Duration(drainConfig.GetTimeoutSeconds()) * time.Second - fmt.Println(timeoutSeconds) drainOpts := &drain.Helper{ DeleteLocalData: true, Force: drainConfig.GetForce(), @@ -299,17 +298,7 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { select { case err := <-r.DrainManager.DrainErrors: - r.Info("failed to cordon/drain targets", "error", err, "scalinggroup", scalingGroupName, "targets", terminateTargets) - maxRetries := *strategy.MaxRetries - if maxRetries > status.GetStrategyRetryCount() { - if maxRetries == -1 { - // if maxRetries is set to -1, retry forever - status.SetStrategyRetryCount(-1) - } else { - // otherwise increment retry counter - status.IncrementStrategyRetryCount() - } - } + r.Info("failed to cordon/drain targets", "error", err, "instancegroup", namespacedName, "targets", terminateTargets) return false, err case <-timeout: diff --git a/controllers/provisioners/eks/update.go b/controllers/provisioners/eks/update.go index 0928cdf1..35de0a19 100644 --- a/controllers/provisioners/eks/update.go +++ b/controllers/provisioners/eks/update.go @@ -84,7 +84,6 @@ func (ctx *EksInstanceGroupContext) Update() error { if err := scalingConfig.Create(config); err != nil { return errors.Wrap(err, "failed to create scaling configuration") } - } if scalingConfig.RotationNeeded(&scaling.DiscoverConfigurationInput{ From 57c1f4a7904767f9d3ab760cdcb5ba2c7f73e6f1 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Thu, 18 Feb 2021 10:13:58 -0800 Subject: [PATCH 08/11] remove readiness gates Signed-off-by: Eytan Avisror --- api/v1alpha1/instancegroup_types.go | 26 +------- api/v1alpha1/zz_generated.deepcopy.go | 20 ------ ...stancemgr.keikoproj.io_instancegroups.yaml | 12 ---- controllers/provisioners/eks/rollingupdate.go | 65 +++++++------------ 4 files changed, 26 insertions(+), 97 deletions(-) diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go index f900bdd2..aa6559bc 100644 --- a/api/v1alpha1/instancegroup_types.go +++ b/api/v1alpha1/instancegroup_types.go @@ -110,9 +110,6 @@ var ( }, } - NamespaceReadinessGateType = "namespace" - AllowedReadinessGateTypes = []string{NamespaceReadinessGateType} - DefaultCRDStrategyMaxRetries = 3 AllowedFileSystemTypes = []string{FileSystemTypeXFS, FileSystemTypeEXT4} @@ -160,13 +157,8 @@ type AwsUpgradeStrategy struct { } type RollingUpdateStrategy struct { - MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` - DrainOptions RollingUpgradeDrainOptions `json:"drainOptions,omitempty"` - ReadinessGates []RollingUpgradeReadinessGate `json:"readinessGates,omitempty"` -} - -func (s *RollingUpdateStrategy) GetReadinessGates() []RollingUpgradeReadinessGate { - return s.ReadinessGates + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + DrainOptions RollingUpgradeDrainOptions `json:"drainOptions,omitempty"` } func (s *RollingUpdateStrategy) GetDrainOptions() RollingUpgradeDrainOptions { @@ -181,11 +173,6 @@ func (s *RollingUpdateStrategy) SetMaxUnavailable(value *intstr.IntOrString) { s.MaxUnavailable = value } -type RollingUpgradeReadinessGate struct { - Type string `json:"type"` - Value string `json:"value"` -} - type RollingUpgradeDrainOptions struct { TimeoutSeconds *int64 `json:"timeoutSeconds,omitempty"` Force *bool `json:"force,omitempty"` @@ -899,15 +886,6 @@ func (r *RollingUpdateStrategy) Validate() error { r.DrainOptions.Force = DefaultRollingUpdateStrategy.DrainOptions.Force } - for _, gate := range r.ReadinessGates { - if !common.ContainsEqualFold(AllowedReadinessGateTypes, gate.Type) { - return errors.Errorf("readiness gate type '%v' is unsupported, supported types are '%v'", gate.Type, AllowedReadinessGateTypes) - } - if common.StringEmpty(gate.Value) { - return errors.New("readiness gate value cannot be empty") - } - } - return nil } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a2618fdb..c5aada4b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -616,11 +616,6 @@ func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { **out = **in } in.DrainOptions.DeepCopyInto(&out.DrainOptions) - if in.ReadinessGates != nil { - in, out := &in.ReadinessGates, &out.ReadinessGates - *out = make([]RollingUpgradeReadinessGate, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStrategy. @@ -658,21 +653,6 @@ func (in *RollingUpgradeDrainOptions) DeepCopy() *RollingUpgradeDrainOptions { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RollingUpgradeReadinessGate) DeepCopyInto(out *RollingUpgradeReadinessGate) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpgradeReadinessGate. -func (in *RollingUpgradeReadinessGate) DeepCopy() *RollingUpgradeReadinessGate { - if in == nil { - return nil - } - out := new(RollingUpgradeReadinessGate) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserDataStage) DeepCopyInto(out *UserDataStage) { *out = *in diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml index 798b2f28..bf469807 100644 --- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml +++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml @@ -392,18 +392,6 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true - readinessGates: - items: - properties: - type: - type: string - value: - type: string - required: - - type - - value - type: object - type: array type: object type: type: string diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index bf8f5ab7..7c5c81f6 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -188,11 +188,10 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { var ( scalingGroupName = aws.StringValue(r.ScalingGroup.AutoScalingGroupName) - instanceCount = len(r.AllInstances) + desiredCount = aws.Int64Value(r.ScalingGroup.DesiredCapacity) namespacedName = r.InstanceGroup.NamespacedName() strategy = r.InstanceGroup.GetUpgradeStrategy().GetRollingUpdateType() drainConfig = strategy.GetDrainOptions() - status = r.InstanceGroup.GetStatus() ) ok := awsprovider.IsDesiredInService(r.ScalingGroup) @@ -201,7 +200,7 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { return false, nil } - ok, err := kubeprovider.IsMinNodesReady(r.ClusterNodes, r.AllInstances, instanceCount) + ok, err := kubeprovider.IsMinNodesReady(r.ClusterNodes, r.AllInstances, int(desiredCount)) if err != nil { return false, err } @@ -211,16 +210,6 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { } r.Info("starting rolling update", "scalinggroup", scalingGroupName, "targets", r.UpdateTargets, "maxunavailable", r.MaxUnavailable) - - ok, err = r.IsReadinessGateAllowed() - if err != nil { - return false, err - } - if !ok { - r.Info("readiness gates are not passing", "scalinggroup", scalingGroupName) - return false, nil - } - if len(r.UpdateTargets) == 0 { r.Info("no updatable instances", "scalinggroup", scalingGroupName) return true, nil @@ -249,24 +238,24 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { r.Info("starting rotation on target nodes", "targets", targetNames) + buff := bytes.NewBufferString("") + timeoutSeconds := time.Duration(drainConfig.GetTimeoutSeconds()) * time.Second + drainOpts := &drain.Helper{ + DeleteLocalData: true, + Force: drainConfig.GetForce(), + IgnoreAllDaemonSets: true, + Timeout: timeoutSeconds, + GracePeriodSeconds: -1, + Client: r.KubernetesClient.Kubernetes, + Out: buff, + ErrOut: buff, + } + // Only create new threads if waitgroup is empty if reflect.DeepEqual(r.DrainManager.DrainGroup, &sync.WaitGroup{}) { for _, node := range targetNodes.Items { nodeName := node.GetName() r.Info("creating drainer goroutine for node", "node", nodeName) - buff := bytes.NewBufferString("") - timeoutSeconds := time.Duration(drainConfig.GetTimeoutSeconds()) * time.Second - drainOpts := &drain.Helper{ - DeleteLocalData: true, - Force: drainConfig.GetForce(), - IgnoreAllDaemonSets: true, - Timeout: timeoutSeconds, - GracePeriodSeconds: -1, - Client: r.KubernetesClient.Kubernetes, - Out: buff, - ErrOut: buff, - } - r.DrainManager.DrainGroup.Add(1) n := node @@ -282,7 +271,6 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { err = drain.RunNodeDrain(drainOpts, nodeName) if err != nil { // If drain has failed, try to uncordon - drain.RunCordonOrUncordon(drainOpts, &n, false) werr := errors.Errorf("drain node failed: %v, %v", err.Error(), buff.String()) r.DrainManager.DrainErrors <- werr } @@ -299,6 +287,15 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { select { case err := <-r.DrainManager.DrainErrors: r.Info("failed to cordon/drain targets", "error", err, "instancegroup", namespacedName, "targets", terminateTargets) + + // try uncordon all targets + for _, node := range targetNodes.Items { + n := node + cErr := drain.RunCordonOrUncordon(drainOpts, &n, false) + if cErr != nil { + r.Info("failed to uncordon node", "error", cErr.Error()) + } + } return false, err case <-timeout: @@ -308,7 +305,6 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { // terminate failures are retryable r.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", scalingGroupName, "targets", terminateTargets) } - status.SetStrategyRetryCount(0) return false, nil case <-time.After(DefaultWaitGroupTimeout): @@ -317,16 +313,3 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { return false, nil } } - -func (r *RollingUpdateRequest) IsReadinessGateAllowed() (bool, error) { - var ( - strategy = r.InstanceGroup.GetUpgradeStrategy().GetRollingUpdateType() - readinessGates = strategy.GetReadinessGates() - ) - - for _, gate := range readinessGates { - _ = gate - } - - return true, nil -} From 2d64abcb78277e9d23fc00dbbbf44add66d49ec4 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Thu, 18 Feb 2021 10:26:16 -0800 Subject: [PATCH 09/11] docs Signed-off-by: Eytan Avisror --- docs/EKS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/EKS.md b/docs/EKS.md index 1a9ac2b4..c7ede4c4 100644 --- a/docs/EKS.md +++ b/docs/EKS.md @@ -255,6 +255,8 @@ instance-manager currently supports two types of upgrade strategy, `rollingUpdat rollingUpdate is a basic implementation of a rolling instance replacement - you can define `maxUnavailable` to some number or percent that represents the desired capacity to be rotated at a time. +Under `drainOptions` you can also define the drain timeout in seconds (defaults to 900), and whether to use force when draining (defaults to true). + ```yaml apiVersion: instancemgr.keikoproj.io/v1alpha1 kind: InstanceGroup @@ -265,6 +267,9 @@ spec: strategy: type: rollingUpdate rollingUpdate: + drainOptions: + timeoutSeconds: 300 + force: false maxUnavailable: 30% ``` From 3b8e1bfff1da6fe607429a2b013509604df8df83 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Fri, 19 Feb 2021 10:29:10 -0800 Subject: [PATCH 10/11] fix unit tests Signed-off-by: Eytan Avisror --- controllers/provisioners/eks/eks_test.go | 1 + controllers/provisioners/eks/rollingupdate.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/provisioners/eks/eks_test.go b/controllers/provisioners/eks/eks_test.go index fbd34f5d..baf2494c 100644 --- a/controllers/provisioners/eks/eks_test.go +++ b/controllers/provisioners/eks/eks_test.go @@ -392,6 +392,7 @@ func MockScalingInstances(nonUpdatable, updatable int) []*autoscaling.Instance { instances := []*autoscaling.Instance{} for i := 0; i < nonUpdatable; i++ { instances = append(instances, &autoscaling.Instance{ + LifecycleState: aws.String(autoscaling.LifecycleStateInService), LaunchConfigurationName: aws.String("some-launch-config"), LaunchTemplate: &autoscaling.LaunchTemplateSpecification{ LaunchTemplateName: aws.String("some-launch-template"), diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index 7c5c81f6..c958e236 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -287,7 +287,6 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { select { case err := <-r.DrainManager.DrainErrors: r.Info("failed to cordon/drain targets", "error", err, "instancegroup", namespacedName, "targets", terminateTargets) - // try uncordon all targets for _, node := range targetNodes.Items { n := node From b32aa231fac3aa1efd07686909dafb125a2e7e3c Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Sun, 28 Mar 2021 12:16:03 -0700 Subject: [PATCH 11/11] terminate in goroutine Signed-off-by: Eytan Avisror --- controllers/providers/kubernetes/utils.go | 4 +++ controllers/provisioners/eks/rollingupdate.go | 29 +++++++------------ 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/controllers/providers/kubernetes/utils.go b/controllers/providers/kubernetes/utils.go index 4d6e3e97..62e520e0 100644 --- a/controllers/providers/kubernetes/utils.go +++ b/controllers/providers/kubernetes/utils.go @@ -128,6 +128,10 @@ func GetNodesByInstance(instanceIds []string, nodes *corev1.NodeList) *corev1.No return nodeList } +func GetNodeInstanceID(node corev1.Node) string { + return common.GetLastElementBy(node.Spec.ProviderID, "/") +} + func IsNodeReady(n corev1.Node) bool { for _, condition := range n.Status.Conditions { if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go index c958e236..b29229b4 100644 --- a/controllers/provisioners/eks/rollingupdate.go +++ b/controllers/provisioners/eks/rollingupdate.go @@ -255,10 +255,10 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { if reflect.DeepEqual(r.DrainManager.DrainGroup, &sync.WaitGroup{}) { for _, node := range targetNodes.Items { nodeName := node.GetName() + instanceID := kubeprovider.GetNodeInstanceID(node) r.Info("creating drainer goroutine for node", "node", nodeName) r.DrainManager.DrainGroup.Add(1) n := node - go func() { defer r.DrainManager.DrainGroup.Done() r.Info("cordoning node", "node", nodeName) @@ -271,41 +271,32 @@ func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { err = drain.RunNodeDrain(drainOpts, nodeName) if err != nil { // If drain has failed, try to uncordon + drain.RunCordonOrUncordon(drainOpts, &n, false) werr := errors.Errorf("drain node failed: %v, %v", err.Error(), buff.String()) r.DrainManager.DrainErrors <- werr } + if err := r.AwsWorker.TerminateScalingInstances([]string{instanceID}); err != nil { + // terminate failures are retryable + r.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", scalingGroupName, "targets", terminateTargets) + } }() } } - timeout := make(chan struct{}) + done := make(chan struct{}) go func() { - defer close(timeout) + defer close(done) r.DrainManager.DrainGroup.Wait() }() select { case err := <-r.DrainManager.DrainErrors: r.Info("failed to cordon/drain targets", "error", err, "instancegroup", namespacedName, "targets", terminateTargets) - // try uncordon all targets - for _, node := range targetNodes.Items { - n := node - cErr := drain.RunCordonOrUncordon(drainOpts, &n, false) - if cErr != nil { - r.Info("failed to uncordon node", "error", cErr.Error()) - } - } return false, err - - case <-timeout: + case <-done: // goroutines completed, terminate and requeue - r.Info("targets drained successfully, terminating", "scalinggroup", scalingGroupName, "targets", terminateTargets) - if err := r.AwsWorker.TerminateScalingInstances(terminateTargets); err != nil { - // terminate failures are retryable - r.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", scalingGroupName, "targets", terminateTargets) - } + r.Info("targets drained successfully", "scalinggroup", scalingGroupName, "targets", terminateTargets) return false, nil - case <-time.After(DefaultWaitGroupTimeout): // goroutines timed out - requeue r.Info("targets still draining", "scalinggroup", scalingGroupName, "targets", terminateTargets)