diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go index 83340026..385bd31f 100644 --- a/api/v1alpha1/instancegroup_types.go +++ b/api/v1alpha1/instancegroup_types.go @@ -106,12 +106,19 @@ var ( LaunchTemplate, } - DefaultRollingUpdateStrategy = &RollingUpdateStrategy{ + DefaultRollingUpgradeTimeoutSeconds int64 = 900 + DefaultRollingUpgradeForce = true + DefaultRollingUpdateStrategy = &RollingUpdateStrategy{ MaxUnavailable: &intstr.IntOrString{ Type: intstr.Int, IntVal: 1, }, + DrainOptions: RollingUpgradeDrainOptions{ + TimeoutSeconds: &DefaultRollingUpgradeTimeoutSeconds, + Force: &DefaultRollingUpgradeForce, + }, } + DefaultCRDStrategyMaxRetries = 3 AllowedContainerRuntimes = []ContainerRuntime{ContainerDRuntime, DockerRuntime} @@ -160,7 +167,12 @@ type AwsUpgradeStrategy struct { } type RollingUpdateStrategy struct { - MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + DrainOptions RollingUpgradeDrainOptions `json:"drainOptions,omitempty"` +} + +func (s *RollingUpdateStrategy) GetDrainOptions() RollingUpgradeDrainOptions { + return s.DrainOptions } func (s *RollingUpdateStrategy) GetMaxUnavailable() *intstr.IntOrString { @@ -171,6 +183,27 @@ func (s *RollingUpdateStrategy) SetMaxUnavailable(value *intstr.IntOrString) { s.MaxUnavailable = value } +type RollingUpgradeDrainOptions struct { + TimeoutSeconds *int64 `json:"timeoutSeconds,omitempty"` + Force *bool `json:"force,omitempty"` +} + +func (d *RollingUpgradeDrainOptions) GetTimeoutSeconds() int64 { + return *d.TimeoutSeconds +} + +func (d *RollingUpgradeDrainOptions) SetTimeoutSeconds(n *int64) { + d.TimeoutSeconds = n +} + +func (d *RollingUpgradeDrainOptions) GetForce() bool { + return *d.Force +} + +func (d *RollingUpgradeDrainOptions) SetForce(condition *bool) { + d.Force = condition +} + type CRDUpdateStrategy struct { Spec string `json:"spec,omitempty"` CRDName string `json:"crdName,omitempty"` @@ -760,12 +793,14 @@ func (ig *InstanceGroup) Validate() error { } } - if strings.EqualFold(s.AwsUpgradeStrategy.Type, RollingUpdateStrategyName) && s.AwsUpgradeStrategy.RollingUpdateType == nil { - s.AwsUpgradeStrategy.RollingUpdateType = DefaultRollingUpdateStrategy + if strings.EqualFold(s.AwsUpgradeStrategy.Type, RollingUpdateStrategyName) { + if err := s.AwsUpgradeStrategy.RollingUpdateType.Validate(); err != nil { + return err + } } - return nil } + func (c *EKSConfiguration) GetRoleName() string { return c.ExistingRoleName } @@ -950,6 +985,26 @@ func (s *AwsUpgradeStrategy) SetCRDType(crd *CRDUpdateStrategy) { s.CRDType = crd } +func (r *RollingUpdateStrategy) Validate() error { + if r == nil { + r = DefaultRollingUpdateStrategy + } + + if r.MaxUnavailable == nil { + r.MaxUnavailable = DefaultRollingUpdateStrategy.MaxUnavailable + } + + if r.DrainOptions.TimeoutSeconds == nil { + r.DrainOptions.TimeoutSeconds = DefaultRollingUpdateStrategy.DrainOptions.TimeoutSeconds + } + + if r.DrainOptions.Force == nil { + r.DrainOptions.Force = DefaultRollingUpdateStrategy.DrainOptions.Force + } + + return nil +} + func (c *CRDUpdateStrategy) Validate() error { if c.GetSpec() == "" { return errors.New("spec is empty") diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 90f591c7..daaf75e5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -640,6 +640,7 @@ func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { *out = new(intstr.IntOrString) **out = **in } + in.DrainOptions.DeepCopyInto(&out.DrainOptions) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStrategy. @@ -652,6 +653,31 @@ func (in *RollingUpdateStrategy) DeepCopy() *RollingUpdateStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpgradeDrainOptions) DeepCopyInto(out *RollingUpgradeDrainOptions) { + *out = *in + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int64) + **out = **in + } + if in.Force != nil { + in, out := &in.Force, &out.Force + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpgradeDrainOptions. +func (in *RollingUpgradeDrainOptions) DeepCopy() *RollingUpgradeDrainOptions { + if in == nil { + return nil + } + out := new(RollingUpgradeDrainOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserDataStage) DeepCopyInto(out *UserDataStage) { *out = *in diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml index 56a02cea..124f8948 100644 --- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml +++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml @@ -403,6 +403,14 @@ spec: type: object rollingUpdate: properties: + drainOptions: + properties: + force: + type: boolean + timeoutSeconds: + format: int64 + type: integer + type: object maxUnavailable: anyOf: - type: integer diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go index e6f57ff4..22c1788d 100644 --- a/controllers/instancegroup_controller.go +++ b/controllers/instancegroup_controller.go @@ -51,7 +51,9 @@ type InstanceGroupReconciler struct { Auth *InstanceGroupAuthenticator ConfigMap *corev1.ConfigMap Namespaces map[string]corev1.Namespace - NamespacesLock *sync.RWMutex + NamespacesLock *sync.Mutex + DrainGroupMapper *sync.Map + DrainErrorMapper *sync.Map ConfigRetention int Metrics *common.MetricsCollector } @@ -132,6 +134,7 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque // set/unset finalizer r.SetFinalizer(instanceGroup) + namespacedName := instanceGroup.NamespacedName() input := provisioners.ProvisionerInput{ AwsWorker: r.Auth.Aws, @@ -149,6 +152,13 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque ) status.SetConfigHash(configHash) + drainGroup, _ := r.DrainGroupMapper.LoadOrStore(namespacedName, &sync.WaitGroup{}) + drainErrs, _ := r.DrainErrorMapper.LoadOrStore(namespacedName, make(chan error)) + input.DrainManager = kubeprovider.DrainManager{ + DrainErrors: drainErrs.(chan error), + DrainGroup: drainGroup.(*sync.WaitGroup), + } + if !reflect.DeepEqual(*r.ConfigMap, corev1.ConfigMap{}) { // Configmap exist - apply defaults/boundaries if namespace is not excluded namespace := instanceGroup.GetNamespace() diff --git a/controllers/providers/aws/predicates.go b/controllers/providers/aws/predicates.go index 14cae4e4..75fb3fea 100644 --- a/controllers/providers/aws/predicates.go +++ b/controllers/providers/aws/predicates.go @@ -16,6 +16,8 @@ limitations under the License. package aws import ( + "strings" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/eks" @@ -127,6 +129,21 @@ func IsUsingMixedInstances(group *autoscaling.Group) bool { return false } +func IsDesiredInService(scalingGroup *autoscaling.Group) bool { + var ( + desired = aws.Int64Value(scalingGroup.DesiredCapacity) + inServiceCount int64 + ) + + for _, instance := range scalingGroup.Instances { + lifecycle := aws.StringValue(instance.LifecycleState) + if strings.EqualFold(lifecycle, autoscaling.LifecycleStateInService) { + inServiceCount++ + } + } + + return desired == inServiceCount + func IsUsingWarmPool(group *autoscaling.Group) bool { if group.WarmPoolConfiguration != nil { return true diff --git a/controllers/providers/kubernetes/rollingupdate.go b/controllers/providers/kubernetes/rollingupdate.go deleted file mode 100644 index e0346ea9..00000000 --- a/controllers/providers/kubernetes/rollingupdate.go +++ /dev/null @@ -1,88 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubernetes - -import ( - awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" - corev1 "k8s.io/api/core/v1" - ctrl "sigs.k8s.io/controller-runtime" -) - -const ( - RollingUpdateStrategyName = "rollingupdate" -) - -var ( - log = ctrl.Log.WithName("kubernetes-provider") -) - -type RollingUpdateRequest struct { - AwsWorker awsprovider.AwsWorker - ClusterNodes *corev1.NodeList - ScalingGroupName string - MaxUnavailable int - DesiredCapacity int - AllInstances []string - UpdateTargets []string -} - -func ProcessRollingUpgradeStrategy(req *RollingUpdateRequest) (bool, error) { - - log.Info("starting rolling update", - "scalinggroup", req.ScalingGroupName, - "targets", req.UpdateTargets, - "maxunavailable", req.MaxUnavailable, - ) - if len(req.UpdateTargets) == 0 { - log.Info("no updatable instances", "scalinggroup", req.ScalingGroupName) - return true, nil - } - - // cannot rotate if maxUnavailable is greater than number of desired - if req.MaxUnavailable > req.DesiredCapacity { - log.Info("maxUnavailable exceeds desired capacity, setting maxUnavailable match desired", - "scalinggroup", req.ScalingGroupName, - "maxunavailable", req.MaxUnavailable, - "desiredcapacity", req.DesiredCapacity, - ) - req.MaxUnavailable = req.DesiredCapacity - } - - ok, err := IsMinNodesReady(req.ClusterNodes, req.AllInstances, req.MaxUnavailable) - if err != nil { - return false, err - } - - if !ok { - log.Info("desired nodes are not ready", "scalinggroup", req.ScalingGroupName) - return false, nil - } - - var terminateTargets []string - if req.MaxUnavailable <= len(req.UpdateTargets) { - terminateTargets = req.UpdateTargets[:req.MaxUnavailable] - } else { - terminateTargets = req.UpdateTargets - } - - log.Info("terminating targets", "scalinggroup", req.ScalingGroupName, "targets", terminateTargets) - if err := req.AwsWorker.TerminateScalingInstances(terminateTargets); err != nil { - // terminate failures are retryable - log.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", req.ScalingGroupName, "targets", terminateTargets) - return false, nil - } - return false, nil -} diff --git a/controllers/providers/kubernetes/utils.go b/controllers/providers/kubernetes/utils.go index 82c1ee3d..6f3f1511 100644 --- a/controllers/providers/kubernetes/utils.go +++ b/controllers/providers/kubernetes/utils.go @@ -22,6 +22,7 @@ import ( "html/template" "reflect" "strings" + "sync" jsonpatch "github.com/evanphx/json-patch" "github.com/ghodss/yaml" @@ -37,14 +38,24 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) +var ( + log = ctrl.Log.WithName("kubernetes-provider") +) + type KubernetesClientSet struct { Kubernetes kubernetes.Interface KubeDynamic dynamic.Interface } +type DrainManager struct { + DrainErrors chan error + DrainGroup *sync.WaitGroup +} + func GetUnstructuredInstanceGroup(instanceGroup *v1alpha1.InstanceGroup) (*unstructured.Unstructured, error) { var obj = &unstructured.Unstructured{} content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(instanceGroup) @@ -60,7 +71,7 @@ func IsDesiredNodesReady(nodes *corev1.NodeList, instanceIds []string, desiredCo return false, nil } - readyInstances := GetReadyNodesByInstance(instanceIds, nodes) + readyInstances := GetReadyNodeNamesByInstance(instanceIds, nodes) // if discovered nodes match provided instance ids, condition is ready if common.StringSliceEquals(readyInstances, instanceIds) { @@ -76,7 +87,7 @@ func IsMinNodesReady(nodes *corev1.NodeList, instanceIds []string, minCount int) return false, nil } - readyInstances := GetReadyNodesByInstance(instanceIds, nodes) + readyInstances := GetReadyNodeNamesByInstance(instanceIds, nodes) // if discovered nodes match provided instance ids, condition is ready if common.StringSliceContains(readyInstances, instanceIds) { @@ -86,7 +97,7 @@ func IsMinNodesReady(nodes *corev1.NodeList, instanceIds []string, minCount int) return false, nil } -func GetReadyNodesByInstance(instanceIds []string, nodes *corev1.NodeList) []string { +func GetReadyNodeNamesByInstance(instanceIds []string, nodes *corev1.NodeList) []string { readyInstances := make([]string, 0) for _, id := range instanceIds { for _, node := range nodes.Items { @@ -98,6 +109,26 @@ func GetReadyNodesByInstance(instanceIds []string, nodes *corev1.NodeList) []str return readyInstances } +func GetNodesByInstance(instanceIds []string, nodes *corev1.NodeList) *corev1.NodeList { + nodeList := &corev1.NodeList{ + ListMeta: metav1.ListMeta{}, + Items: []corev1.Node{}, + } + + for _, id := range instanceIds { + for _, node := range nodes.Items { + if common.GetLastElementBy(node.Spec.ProviderID, "/") == id { + nodeList.Items = append(nodeList.Items, node) + } + } + } + return nodeList +} + +func GetNodeInstanceID(node corev1.Node) string { + return common.GetLastElementBy(node.Spec.ProviderID, "/") +} + func IsNodeReady(n corev1.Node) bool { for _, condition := range n.Status.Conditions { if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go index dd15a5c1..ad50a604 100644 --- a/controllers/provisioners/eks/eks.go +++ b/controllers/provisioners/eks/eks.go @@ -19,15 +19,13 @@ import ( "fmt" "sync" - corev1 "k8s.io/api/core/v1" - "github.com/go-logr/logr" - "github.com/keikoproj/instance-manager/api/v1alpha1" "github.com/keikoproj/instance-manager/controllers/common" awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" "github.com/keikoproj/instance-manager/controllers/provisioners" + corev1 "k8s.io/api/core/v1" ) const ( @@ -72,6 +70,7 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext { ctx := &EksInstanceGroupContext{ InstanceGroup: instanceGroup, + DrainManager: p.DrainManager, KubernetesClient: p.Kubernetes, AwsWorker: p.AwsWorker, Log: p.Log.WithName("eks"), @@ -89,6 +88,7 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext { type EksInstanceGroupContext struct { sync.Mutex + DrainManager kubeprovider.DrainManager InstanceGroup *v1alpha1.InstanceGroup KubernetesClient kubeprovider.KubernetesClientSet AwsWorker awsprovider.AwsWorker diff --git a/controllers/provisioners/eks/eks_test.go b/controllers/provisioners/eks/eks_test.go index a50c1c60..bb5e28eb 100644 --- a/controllers/provisioners/eks/eks_test.go +++ b/controllers/provisioners/eks/eks_test.go @@ -413,7 +413,7 @@ func MockAwsCRDStrategy(spec string) v1alpha1.AwsUpgradeStrategy { func MockAwsRollingUpdateStrategy(maxUnavailable *intstr.IntOrString) v1alpha1.AwsUpgradeStrategy { return v1alpha1.AwsUpgradeStrategy{ - Type: kubeprovider.RollingUpdateStrategyName, + Type: RollingUpdateStrategyName, RollingUpdateType: &v1alpha1.RollingUpdateStrategy{ MaxUnavailable: maxUnavailable, }, @@ -424,6 +424,7 @@ func MockScalingInstances(nonUpdatable, updatable int) []*autoscaling.Instance { instances := []*autoscaling.Instance{} for i := 0; i < nonUpdatable; i++ { instances = append(instances, &autoscaling.Instance{ + LifecycleState: aws.String(autoscaling.LifecycleStateInService), LaunchConfigurationName: aws.String("some-launch-config"), LaunchTemplate: &autoscaling.LaunchTemplateSpecification{ LaunchTemplateName: aws.String("some-launch-template"), diff --git a/controllers/provisioners/eks/rollingupdate.go b/controllers/provisioners/eks/rollingupdate.go new file mode 100644 index 00000000..b29229b4 --- /dev/null +++ b/controllers/provisioners/eks/rollingupdate.go @@ -0,0 +1,305 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eks + +import ( + "bytes" + "reflect" + "sort" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/go-logr/logr" + "github.com/keikoproj/instance-manager/api/v1alpha1" + "github.com/keikoproj/instance-manager/controllers/common" + awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" + kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" + "github.com/keikoproj/instance-manager/controllers/provisioners/eks/scaling" + "github.com/pkg/errors" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + drain "k8s.io/kubectl/pkg/drain" +) + +const ( + RollingUpdateStrategyName = "rollingupdate" +) + +var ( + DefaultWaitGroupTimeout = time.Second * 5 + TransientLifecycleStates = []string{ + autoscaling.LifecycleStateDetaching, + autoscaling.LifecycleStateEnteringStandby, + autoscaling.LifecycleStatePending, + autoscaling.LifecycleStatePendingProceed, + autoscaling.LifecycleStatePendingWait, + autoscaling.LifecycleStateTerminating, + autoscaling.LifecycleStateTerminatingWait, + autoscaling.LifecycleStateTerminatingProceed, + autoscaling.LifecycleStateTerminated, + } +) + +type RollingUpdateRequest struct { + logr.Logger + InstanceGroup *v1alpha1.InstanceGroup + AwsWorker awsprovider.AwsWorker + KubernetesClient kubeprovider.KubernetesClientSet + ClusterNodes *corev1.NodeList + ScalingGroup *autoscaling.Group + DrainManager kubeprovider.DrainManager + MaxUnavailable int + DesiredCapacity int + AllInstances []string + UpdateTargets []string +} + +func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *RollingUpdateRequest { + var ( + needsUpdate []string + allInstances []string + instanceGroup = ctx.GetInstanceGroup() + state = ctx.GetDiscoveredState() + scalingConfig = state.GetScalingConfiguration() + scalingResource = scalingConfig.Resource() + scalingGroup = state.GetScalingGroup() + desiredCount = int(aws.Int64Value(scalingGroup.DesiredCapacity)) + strategy = instanceGroup.GetUpgradeStrategy().GetRollingUpdateType() + maxUnavailable = strategy.GetMaxUnavailable() + ) + + // Get all Autoscaling Instances that needs update + for _, instance := range scalingGroup.Instances { + var ( + instanceId = aws.StringValue(instance.InstanceId) + lifecycle = aws.StringValue(instance.LifecycleState) + ) + + if common.ContainsEqualFold(TransientLifecycleStates, lifecycle) { + continue + } + + allInstances = append(allInstances, aws.StringValue(instance.InstanceId)) + + if awsprovider.IsUsingLaunchConfiguration(scalingGroup) { + if instance.LaunchConfigurationName == nil { + needsUpdate = append(needsUpdate, instanceId) + continue + } + + var ( + config = aws.StringValue(instance.LaunchConfigurationName) + activeConfig = aws.StringValue(scalingGroup.LaunchConfigurationName) + ) + + if !strings.EqualFold(config, activeConfig) { + needsUpdate = append(needsUpdate, instanceId) + } + } + + if awsprovider.IsUsingLaunchTemplate(scalingGroup) { + if instance.LaunchTemplate == nil { + needsUpdate = append(needsUpdate, instanceId) + continue + } + + var ( + config = aws.StringValue(instance.LaunchTemplate.LaunchTemplateName) + version = aws.StringValue(instance.LaunchTemplate.Version) + launchTemplate = scaling.ConvertToLaunchTemplate(scalingResource) + activeConfig = aws.StringValue(scalingGroup.LaunchTemplate.LaunchTemplateName) + activeVersionNum = aws.Int64Value(launchTemplate.LatestVersionNumber) + activeVersion = common.Int64ToStr(activeVersionNum) + ) + if !strings.EqualFold(config, activeConfig) || !strings.EqualFold(version, activeVersion) { + needsUpdate = append(needsUpdate, instanceId) + } + } + + if awsprovider.IsUsingMixedInstances(scalingGroup) { + if instance.LaunchTemplate == nil { + needsUpdate = append(needsUpdate, instanceId) + continue + } + + var ( + config = aws.StringValue(instance.LaunchTemplate.LaunchTemplateName) + version = aws.StringValue(instance.LaunchTemplate.Version) + launchTemplate = scaling.ConvertToLaunchTemplate(scalingResource) + activeConfig = aws.StringValue(scalingGroup.MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateName) + activeVersionNum = aws.Int64Value(launchTemplate.LatestVersionNumber) + activeVersion = common.Int64ToStr(activeVersionNum) + ) + + if !strings.EqualFold(config, activeConfig) || !strings.EqualFold(version, activeVersion) { + needsUpdate = append(needsUpdate, instanceId) + } + } + + } + allCount := len(allInstances) + + var unavailableInt int + if maxUnavailable.Type == intstr.String { + unavailableInt, _ = intstr.GetValueFromIntOrPercent(maxUnavailable, allCount, true) + } else { + unavailableInt = maxUnavailable.IntValue() + } + + if unavailableInt == 0 { + unavailableInt = 1 + } + + sort.Strings(needsUpdate) + + return &RollingUpdateRequest{ + Logger: ctx.Log, + InstanceGroup: instanceGroup, + KubernetesClient: ctx.KubernetesClient, + AwsWorker: ctx.AwsWorker, + DrainManager: ctx.DrainManager, + ClusterNodes: state.GetClusterNodes(), + MaxUnavailable: unavailableInt, + DesiredCapacity: desiredCount, + AllInstances: allInstances, + UpdateTargets: needsUpdate, + ScalingGroup: scalingGroup, + } +} + +func (r *RollingUpdateRequest) ProcessRollingUpgradeStrategy() (bool, error) { + + var ( + scalingGroupName = aws.StringValue(r.ScalingGroup.AutoScalingGroupName) + desiredCount = aws.Int64Value(r.ScalingGroup.DesiredCapacity) + namespacedName = r.InstanceGroup.NamespacedName() + strategy = r.InstanceGroup.GetUpgradeStrategy().GetRollingUpdateType() + drainConfig = strategy.GetDrainOptions() + ) + + ok := awsprovider.IsDesiredInService(r.ScalingGroup) + if !ok { + r.Info("desired instances are not in service", "scalinggroup", scalingGroupName) + return false, nil + } + + ok, err := kubeprovider.IsMinNodesReady(r.ClusterNodes, r.AllInstances, int(desiredCount)) + if err != nil { + return false, err + } + if !ok { + r.Info("desired nodes are not ready", "scalinggroup", scalingGroupName) + return false, nil + } + + r.Info("starting rolling update", "scalinggroup", scalingGroupName, "targets", r.UpdateTargets, "maxunavailable", r.MaxUnavailable) + if len(r.UpdateTargets) == 0 { + r.Info("no updatable instances", "scalinggroup", scalingGroupName) + return true, nil + } + + // cannot rotate if maxUnavailable is greater than number of desired + if r.MaxUnavailable > r.DesiredCapacity { + r.Info("maxUnavailable exceeds desired capacity, setting maxUnavailable match desired", + "scalinggroup", scalingGroupName, "maxunavailable", r.MaxUnavailable, "desiredcapacity", r.DesiredCapacity) + r.MaxUnavailable = r.DesiredCapacity + } + + var terminateTargets []string + if r.MaxUnavailable <= len(r.UpdateTargets) { + terminateTargets = r.UpdateTargets[:r.MaxUnavailable] + } else { + terminateTargets = r.UpdateTargets + } + + targetNodes := kubeprovider.GetNodesByInstance(terminateTargets, r.ClusterNodes) + + targetNames := make([]string, 0) + for _, node := range targetNodes.Items { + targetNames = append(targetNames, node.GetName()) + } + + r.Info("starting rotation on target nodes", "targets", targetNames) + + buff := bytes.NewBufferString("") + timeoutSeconds := time.Duration(drainConfig.GetTimeoutSeconds()) * time.Second + drainOpts := &drain.Helper{ + DeleteLocalData: true, + Force: drainConfig.GetForce(), + IgnoreAllDaemonSets: true, + Timeout: timeoutSeconds, + GracePeriodSeconds: -1, + Client: r.KubernetesClient.Kubernetes, + Out: buff, + ErrOut: buff, + } + + // Only create new threads if waitgroup is empty + if reflect.DeepEqual(r.DrainManager.DrainGroup, &sync.WaitGroup{}) { + for _, node := range targetNodes.Items { + nodeName := node.GetName() + instanceID := kubeprovider.GetNodeInstanceID(node) + r.Info("creating drainer goroutine for node", "node", nodeName) + r.DrainManager.DrainGroup.Add(1) + n := node + go func() { + defer r.DrainManager.DrainGroup.Done() + r.Info("cordoning node", "node", nodeName) + err := drain.RunCordonOrUncordon(drainOpts, &n, true) + if err != nil { + werr := errors.Errorf("cordon node failed: %v, %v", err.Error(), buff.String()) + r.DrainManager.DrainErrors <- werr + } + r.Info("draining node", "node", nodeName) + err = drain.RunNodeDrain(drainOpts, nodeName) + if err != nil { + // If drain has failed, try to uncordon + drain.RunCordonOrUncordon(drainOpts, &n, false) + werr := errors.Errorf("drain node failed: %v, %v", err.Error(), buff.String()) + r.DrainManager.DrainErrors <- werr + } + if err := r.AwsWorker.TerminateScalingInstances([]string{instanceID}); err != nil { + // terminate failures are retryable + r.Info("failed to terminate targets", "reason", err.Error(), "scalinggroup", scalingGroupName, "targets", terminateTargets) + } + }() + } + } + + done := make(chan struct{}) + go func() { + defer close(done) + r.DrainManager.DrainGroup.Wait() + }() + + select { + case err := <-r.DrainManager.DrainErrors: + r.Info("failed to cordon/drain targets", "error", err, "instancegroup", namespacedName, "targets", terminateTargets) + return false, err + case <-done: + // goroutines completed, terminate and requeue + r.Info("targets drained successfully", "scalinggroup", scalingGroupName, "targets", terminateTargets) + return false, nil + case <-time.After(DefaultWaitGroupTimeout): + // goroutines timed out - requeue + r.Info("targets still draining", "scalinggroup", scalingGroupName, "targets", terminateTargets) + return false, nil + } +} diff --git a/controllers/provisioners/eks/update.go b/controllers/provisioners/eks/update.go index 34b854fc..dfb86d50 100644 --- a/controllers/provisioners/eks/update.go +++ b/controllers/provisioners/eks/update.go @@ -88,7 +88,6 @@ func (ctx *EksInstanceGroupContext) Update() error { if err := scalingConfig.Create(config); err != nil { return errors.Wrap(err, "failed to create scaling configuration") } - } if scalingConfig.RotationNeeded(&scaling.DiscoverConfigurationInput{ diff --git a/controllers/provisioners/eks/upgrade.go b/controllers/provisioners/eks/upgrade.go index 8d4b944d..434c0ff8 100644 --- a/controllers/provisioners/eks/upgrade.go +++ b/controllers/provisioners/eks/upgrade.go @@ -26,7 +26,6 @@ import ( kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" "github.com/keikoproj/instance-manager/controllers/provisioners/eks/scaling" "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/intstr" ) func (ctx *EksInstanceGroupContext) UpgradeNodes() error { @@ -63,9 +62,9 @@ func (ctx *EksInstanceGroupContext) UpgradeNodes() error { break } return nil - case kubeprovider.RollingUpdateStrategyName: + case RollingUpdateStrategyName: req := ctx.NewRollingUpdateRequest() - ok, err := kubeprovider.ProcessRollingUpgradeStrategy(req) + ok, err := req.ProcessRollingUpgradeStrategy() if err != nil { state.Publisher.Publish(kubeprovider.InstanceGroupUpgradeFailedEvent, "instancegroup", instanceGroup.NamespacedName(), "type", kubeprovider.RollingUpdateStrategyName, "error", err) ctx.SetState(v1alpha1.ReconcileErr) @@ -261,4 +260,4 @@ func (ctx *EksInstanceGroupContext) NewRollingUpdateRequest() *kubeprovider.Roll UpdateTargets: needsUpdate, ScalingGroupName: asgName, } -} +} \ No newline at end of file diff --git a/controllers/provisioners/provisioners.go b/controllers/provisioners/provisioners.go index b2985ee7..da619764 100644 --- a/controllers/provisioners/provisioners.go +++ b/controllers/provisioners/provisioners.go @@ -27,6 +27,7 @@ const ( ) type ProvisionerInput struct { + DrainManager kubeprovider.DrainManager AwsWorker awsprovider.AwsWorker Kubernetes kubeprovider.KubernetesClientSet InstanceGroup *v1alpha1.InstanceGroup diff --git a/docs/EKS.md b/docs/EKS.md index 2e4dfec4..c5592a14 100644 --- a/docs/EKS.md +++ b/docs/EKS.md @@ -270,6 +270,8 @@ instance-manager currently supports two types of upgrade strategy, `rollingUpdat rollingUpdate is a basic implementation of a rolling instance replacement - you can define `maxUnavailable` to some number or percent that represents the desired capacity to be rotated at a time. +Under `drainOptions` you can also define the drain timeout in seconds (defaults to 900), and whether to use force when draining (defaults to true). + ```yaml apiVersion: instancemgr.keikoproj.io/v1alpha1 kind: InstanceGroup @@ -280,6 +282,9 @@ spec: strategy: type: rollingUpdate rollingUpdate: + drainOptions: + timeoutSeconds: 300 + force: false maxUnavailable: 30% ``` diff --git a/go.mod b/go.mod index 3d194e3c..6b9c99ca 100644 --- a/go.mod +++ b/go.mod @@ -20,5 +20,6 @@ require ( k8s.io/api v0.19.6 k8s.io/apimachinery v0.19.6 k8s.io/client-go v0.19.6 + k8s.io/kubectl v0.19.6 sigs.k8s.io/controller-runtime v0.7.0 ) diff --git a/go.sum b/go.sum index 21adef42..5522e642 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= @@ -27,6 +28,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -34,8 +37,10 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -76,6 +81,7 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -102,20 +108,25 @@ github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAEl github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -123,6 +134,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= @@ -159,11 +174,13 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -177,6 +194,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -186,6 +204,7 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -219,10 +238,14 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= +github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -253,6 +276,7 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -301,7 +325,6 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -331,6 +354,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -340,6 +366,7 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -352,11 +379,14 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -396,6 +426,10 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -458,6 +492,7 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -511,12 +546,12 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -750,7 +785,6 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -790,6 +824,7 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -808,10 +843,17 @@ k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlm k8s.io/apimachinery v0.19.6 h1:kBLzSGuDdY1NdSV2uFzI+FwZ9wtkmG+X3ZVcWXSqNgA= k8s.io/apimachinery v0.19.6/go.mod h1:6sRbGRAVY5DOCuZwB5XkqguBqpqLU6q/kOaOdk29z6Q= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= +k8s.io/cli-runtime v0.19.6 h1:PwRKKVbCFzcGJ9WxctV6liI95GMiAneJrSFZTr6Za94= +k8s.io/cli-runtime v0.19.6/go.mod h1:VOnivMtWLPKVuFBEVS9jM1WbR9TynCIja9H5jwviY/c= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/client-go v0.19.6 h1:vtPb33nP8DBMW+/CyuJ8fiie36c3CM1Ts6L4Tsr+PtU= k8s.io/client-go v0.19.6/go.mod h1:gEiS+efRlXYUEQ9Oz4lmNXlxAl5JZ8y2zbTDGhvXXnk= k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/code-generator v0.19.6/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= +k8s.io/component-base v0.19.2 h1:jW5Y9RcZTb79liEhW3XDVTW7MuvEGP0tQZnfSX6/+gs= +k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= +k8s.io/component-base v0.19.6 h1:V76d3rIEWvP95peWgRycKslQnEwlaPy4UORvh3+YBbU= +k8s.io/component-base v0.19.6/go.mod h1:8Btsf8J00/fVDa/YFmXjei7gVkcFrlKZXjSeP4SZNJg= k8s.io/component-base v0.19.2 h1:jW5Y9RcZTb79liEhW3XDVTW7MuvEGP0tQZnfSX6/+gs= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -821,6 +863,10 @@ k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kubectl v0.19.6 h1:7FpynLRAyCuvVQtoV/bwwNKcwZTnprVHc6fui57KiRc= +k8s.io/kubectl v0.19.6/go.mod h1:SenrDhlFxxBHnf4amqNPY2ajWJ3CDMhrcm3sCOY4pLI= +k8s.io/metrics v0.19.6/go.mod h1:jM61saf/bjMRmow6zan2cAk8vFDmqvbNXFRbB4g7TNs= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -828,9 +874,12 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= +sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= \ No newline at end of file diff --git a/main.go b/main.go index fbc4151c..2ed14ad7 100644 --- a/main.go +++ b/main.go @@ -158,6 +158,8 @@ func main() { Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("instancegroup"), MaxParallel: maxParallel, + DrainGroupMapper: &sync.Map{}, + DrainErrorMapper: &sync.Map{}, Auth: &controllers.InstanceGroupAuthenticator{ Aws: awsWorker, Kubernetes: kube,