From a37e4398b8e333c6f4647369c9485faa00f5d61a Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 28 Sep 2021 06:55:55 +0000 Subject: [PATCH 1/3] Knob to disable/enable leaked eni cleanup --- README.md | 8 +++++++- pkg/awsutils/awsutils.go | 14 ++++++++++--- pkg/awsutils/awsutils_test.go | 38 +++++++++++++++++++++++++++++++---- pkg/ipamd/ipamd.go | 11 +++++++++- 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 86f2fae57a..d598c6a093 100644 --- a/README.md +++ b/README.md @@ -522,10 +522,12 @@ This plugin interacts with the following tags on ENIs: * `cluster.k8s.amazonaws.com/name` * `node.k8s.amazonaws.com/instance_id` * `node.k8s.amazonaws.com/no_manage` +* `kubernetes.io/cluster/` +* `eks:eni:owner` #### Cluster Name tag -The tag `cluster.k8s.amazonaws.com/name` will be set to the cluster name of the +The tag `cluster.k8s.amazonaws.com/name` and `kubernetes.io/cluster/` will be set to the cluster name of the aws-node daemonset which created the ENI. #### Instance ID tag @@ -533,6 +535,10 @@ aws-node daemonset which created the ENI. The tag `node.k8s.amazonaws.com/instance_id` will be set to the instance ID of the aws-node instance that allocated this ENI. +#### ENI owner tag + +The tag `eks:eni:owner` will be set to `amazon-vpc-cni` since aws-node created this ENI. + #### No Manage tag The tag `node.k8s.amazonaws.com/no_manage` is read by the aws-node daemonset to diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index dd44c455ba..2bbfba97cc 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -54,6 +54,11 @@ const ( eniClusterTagKey = "cluster.k8s.amazonaws.com/name" additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS" reservedTagKeyPrefix = "k8s.amazonaws.com" + clusterNameTagKeyFormat = "kubernetes.io/cluster/%s" + clusterNameTagValue = "owned" + + networkInterfaceOwnerTagKey = "eks:eni:owner" + networkInterfaceOwnerTagValue = "amazon-vpc-cni" // UnknownInstanceType indicates that the instance type is not yet supported UnknownInstanceType = "vpc ip resource(eni ip limit): unknown instance type" @@ -371,7 +376,7 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string) } // New creates an EC2InstanceMetadataCache -func New(useCustomNetworking, disableENIProvisioning, v4Enabled, v6Enabled bool) (*EC2InstanceMetadataCache, error) { +func New(useCustomNetworking, disableENIProvisioning, v4Enabled, v6Enabled, disableLeakedENICollection bool) (*EC2InstanceMetadataCache, error) { //ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run ctx := context.Background() @@ -411,7 +416,7 @@ func New(useCustomNetworking, disableENIProvisioning, v4Enabled, v6Enabled bool) } // Clean up leaked ENIs in the background - if !disableENIProvisioning { + if !disableENIProvisioning && !disableLeakedENICollection { go wait.Forever(cache.cleanUpLeakedENIs, time.Hour) } @@ -828,13 +833,16 @@ func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string // buildENITags computes the desired AWS Tags for eni func (cache *EC2InstanceMetadataCache) buildENITags() map[string]string { tags := map[string]string{ - eniNodeTagKey: cache.instanceID, + eniNodeTagKey: cache.instanceID, + networkInterfaceOwnerTagKey: networkInterfaceOwnerTagValue, } // If clusterName is provided, // tag the ENI with "cluster.k8s.amazonaws.com/name=" + // and "kubernetes.io/cluster/: owned" if cache.clusterName != "" { tags[eniClusterTagKey] = cache.clusterName + tags[fmt.Sprintf(clusterNameTagKeyFormat, cache.clusterName)] = clusterNameTagValue } for key, value := range cache.additionalENITags { tags[key] = value diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index c190402a99..34a64c234e 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -914,6 +914,7 @@ func TestEC2InstanceMetadataCache_buildENITags(t *testing.T) { }, want: map[string]string{ "node.k8s.amazonaws.com/instance_id": "i-xxxxx", + "eks:eni:owner": "amazon-vpc-cni", }, }, { @@ -923,8 +924,10 @@ func TestEC2InstanceMetadataCache_buildENITags(t *testing.T) { clusterName: "awesome-cluster", }, want: map[string]string{ - "node.k8s.amazonaws.com/instance_id": "i-xxxxx", - "cluster.k8s.amazonaws.com/name": "awesome-cluster", + "node.k8s.amazonaws.com/instance_id": "i-xxxxx", + "cluster.k8s.amazonaws.com/name": "awesome-cluster", + "kubernetes.io/cluster/awesome-cluster": "owned", + "eks:eni:owner": "amazon-vpc-cni", }, }, { @@ -940,6 +943,7 @@ func TestEC2InstanceMetadataCache_buildENITags(t *testing.T) { "node.k8s.amazonaws.com/instance_id": "i-xxxxx", "tagKey-1": "tagVal-1", "tagKey-2": "tagVal-2", + "eks:eni:owner": "amazon-vpc-cni", }, }, } @@ -1430,6 +1434,14 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { Key: aws.String("cluster.k8s.amazonaws.com/name"), Value: aws.String("awesome-cluster"), }, + { + Key: aws.String("eks:eni:owner"), + Value: aws.String("amazon-vpc-cni"), + }, + { + Key: aws.String("kubernetes.io/cluster/awesome-cluster"), + Value: aws.String("owned"), + }, { Key: aws.String("node.k8s.amazonaws.com/instance_id"), Value: aws.String("i-xxxx"), @@ -1455,8 +1467,10 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { args: args{ eniID: "eni-xxxx", currentTags: map[string]string{ - "node.k8s.amazonaws.com/instance_id": "i-xxxx", - "cluster.k8s.amazonaws.com/name": "awesome-cluster", + "node.k8s.amazonaws.com/instance_id": "i-xxxx", + "cluster.k8s.amazonaws.com/name": "awesome-cluster", + "kubernetes.io/cluster/awesome-cluster": "owned", + "eks:eni:owner": "amazon-vpc-cni", }, }, wantErr: nil, @@ -1475,6 +1489,14 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { Key: aws.String("cluster.k8s.amazonaws.com/name"), Value: aws.String("awesome-cluster"), }, + { + Key: aws.String("eks:eni:owner"), + Value: aws.String("amazon-vpc-cni"), + }, + { + Key: aws.String("kubernetes.io/cluster/awesome-cluster"), + Value: aws.String("owned"), + }, }, }, }, @@ -1503,6 +1525,14 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { Key: aws.String("cluster.k8s.amazonaws.com/name"), Value: aws.String("awesome-cluster"), }, + { + Key: aws.String("eks:eni:owner"), + Value: aws.String("amazon-vpc-cni"), + }, + { + Key: aws.String("kubernetes.io/cluster/awesome-cluster"), + Value: aws.String("owned"), + }, { Key: aws.String("node.k8s.amazonaws.com/instance_id"), Value: aws.String("i-xxxx"), diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 39b4382bab..7d4ef9fe3f 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -156,6 +156,9 @@ const ( envManageUntaggedENI = "MANAGE_UNTAGGED_ENI" eniNodeTagKey = "node.k8s.amazonaws.com/instance_id" + + //envDisableLeakedENICollection is used to disable leaked eni go routine in aws-node + envDisableLeakedENICollection = "DISABLE_LEAKED_ENI_COLLECTION" ) var log = logger.Get() @@ -248,6 +251,7 @@ type IPAMContext struct { enablePrefixDelegation bool lastInsufficientCidrError time.Time enableManageUntaggedMode bool + disableLeakedENICollection bool } // setUnmanagedENIs will rebuild the set of ENI IDs for ENIs tagged as "no_manage" @@ -359,8 +363,9 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex c.enableIPv6 = isIPv6Enabled() c.disableENIProvisioning = disablingENIProvisioning() + c.disableLeakedENICollection = isLeakedENICollectionDisabled() - client, err := awsutils.New(c.useCustomNetworking, c.disableENIProvisioning, c.enableIPv4, c.enableIPv6) + client, err := awsutils.New(c.useCustomNetworking, c.disableENIProvisioning, c.enableIPv4, c.enableIPv6, c.disableLeakedENICollection) if err != nil { return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface") } @@ -1687,6 +1692,10 @@ func enableManageUntaggedMode() bool { return getEnvBoolWithDefault(envManageUntaggedENI, true) } +func isLeakedENICollectionDisabled() bool { + return getEnvBoolWithDefault(envDisableENIProvisioning, false) +} + // filterUnmanagedENIs filters out ENIs marked with the "node.k8s.amazonaws.com/no_manage" tag func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutils.ENIMetadata { numFiltered := 0 From 83b6c18190f1e8221fdb55376aa08f9302039009 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 28 Sep 2021 07:08:20 +0000 Subject: [PATCH 2/3] formatting --- pkg/ipamd/ipamd.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 7d4ef9fe3f..af45f844ea 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -243,14 +243,14 @@ type IPAMContext struct { lastDecreaseIPPool time.Time // reconcileCooldownCache keeps timestamps of the last time an IP address was unassigned from an ENI, // so that we don't reconcile and add it back too quickly if IMDS lags behind reality. - reconcileCooldownCache ReconcileCooldownCache - terminating int32 // Flag to warn that the pod is about to shut down. - disableENIProvisioning bool - enablePodENI bool - myNodeName string - enablePrefixDelegation bool - lastInsufficientCidrError time.Time - enableManageUntaggedMode bool + reconcileCooldownCache ReconcileCooldownCache + terminating int32 // Flag to warn that the pod is about to shut down. + disableENIProvisioning bool + enablePodENI bool + myNodeName string + enablePrefixDelegation bool + lastInsufficientCidrError time.Time + enableManageUntaggedMode bool disableLeakedENICollection bool } From 397b82678a4451920b02683eb836187fc60a8205 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Mon, 24 Jan 2022 16:07:50 +0000 Subject: [PATCH 3/3] fix formatting --- pkg/ipamd/ipamd.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 69b40a2c38..895cca4add 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -168,7 +168,6 @@ const ( // Present and set to the empty string, which we use to mean "CNI DEL had occurred; networking has been removed from this pod" // The empty string one helps close a trace at pod shutdown where it looks like the pod still has its IP when the IP has been released envAnnotatePodIP = "ANNOTATE_POD_IP" - ) var log = logger.Get() @@ -262,8 +261,7 @@ type IPAMContext struct { lastInsufficientCidrError time.Time enableManageUntaggedMode bool disableLeakedENICollection bool - enablePodIPAnnotation bool - + enablePodIPAnnotation bool } // setUnmanagedENIs will rebuild the set of ENI IDs for ENIs tagged as "no_manage" @@ -1695,7 +1693,6 @@ func enableManageUntaggedMode() bool { return getEnvBoolWithDefault(envManageUntaggedENI, true) } - func isLeakedENICollectionDisabled() bool { return getEnvBoolWithDefault(envDisableENIProvisioning, false) }