Add vlan support to ipamd

cmd/cni-metrics-helper/README.md

@@ -15,10 +15,10 @@ The following diagram shows how `cni-metrics-helper` works in a cluster:
 
 ### Installing the cni-metrics-helper
 ```
-kubectl apply -f v1.5/cni-metrics-helper.yaml
+kubectl apply -f v1.6/cni-metrics-helper.yaml
 ```
 
-Adding this will publish the following metrics to CloudWatch:
+Adding the CNI metrics helper will publish the following metrics to CloudWatch:
 ```
 "addReqCount",
 "assignIPAddresses",
@@ -31,6 +31,7 @@ Adding this will publish the following metrics to CloudWatch:
 "ipamdActionInProgress",
 "ipamdErr",
 "maxIPAddresses",
+"podENIErr",
 "reconcileCount",
 "totalIPAddresses",
 ```

cmd/cni-metrics-helper/metrics/cni_metrics.go

@@ -128,6 +128,13 @@ var InterestingCNIMetrics = map[string]metricsConvert{
  actionFunc: metricsAdd,
  data: &dataPoints{},
  logToFile: true}}},
+ "awscni_pod_eni_error_count": {
+ actions: []metricsAction{
+ {cwMetricName: "podENIErr",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
 }
 
 // CNIMetricsTarget defines data structure for kube-state-metric target

config/master/aws-k8s-cni-cn.yaml

@@ -29,12 +29,20 @@
  - ""
  "resources":
  - "pods"
- - "nodes"
  - "namespaces"
  "verbs":
  - "list"
  - "watch"
  - "get"
+- "apiGroups":
+ - ""
+ "resources":
+ - "nodes"
+ "verbs":
+ - "list"
+ - "watch"
+ - "get"
+ - "update"
 - "apiGroups":
  - "extensions"
  "resources":
@@ -135,6 +143,8 @@
  "value": "false"
  - "name": "DISABLE_METRICS"
  "value": "false"
+ - "name": "ENABLE_POD_ENI"
+ "value": "false"
  - "name": "MY_NODE_NAME"
  "valueFrom":
  "fieldRef":

config/master/aws-k8s-cni-us-gov-east-1.yaml

@@ -29,12 +29,20 @@
  - ""
  "resources":
  - "pods"
- - "nodes"
  - "namespaces"
  "verbs":
  - "list"
  - "watch"
  - "get"
+- "apiGroups":
+ - ""
+ "resources":
+ - "nodes"
+ "verbs":
+ - "list"
+ - "watch"
+ - "get"
+ - "update"
 - "apiGroups":
  - "extensions"
  "resources":
@@ -135,6 +143,8 @@
  "value": "false"
  - "name": "DISABLE_METRICS"
  "value": "false"
+ - "name": "ENABLE_POD_ENI"
+ "value": "false"
  - "name": "MY_NODE_NAME"
  "valueFrom":
  "fieldRef":

config/master/aws-k8s-cni-us-gov-west-1.yaml

@@ -29,12 +29,20 @@
  - ""
  "resources":
  - "pods"
- - "nodes"
  - "namespaces"
  "verbs":
  - "list"
  - "watch"
  - "get"
+- "apiGroups":
+ - ""
+ "resources":
+ - "nodes"
+ "verbs":
+ - "list"
+ - "watch"
+ - "get"
+ - "update"
 - "apiGroups":
  - "extensions"
  "resources":
@@ -135,6 +143,8 @@
  "value": "false"
  - "name": "DISABLE_METRICS"
  "value": "false"
+ - "name": "ENABLE_POD_ENI"
+ "value": "false"
  - "name": "MY_NODE_NAME"
  "valueFrom":
  "fieldRef":

config/master/aws-k8s-cni.yaml

@@ -29,12 +29,20 @@
  - ""
  "resources":
  - "pods"
- - "nodes"
  - "namespaces"
  "verbs":
  - "list"
  - "watch"
  - "get"
+- "apiGroups":
+ - ""
+ "resources":
+ - "nodes"
+ "verbs":
+ - "list"
+ - "watch"
+ - "get"
+ - "update"
 - "apiGroups":
  - "extensions"
  "resources":
@@ -135,6 +143,8 @@
  "value": "false"
  - "name": "DISABLE_METRICS"
  "value": "false"
+ - "name": "ENABLE_POD_ENI"
+ "value": "false"
  - "name": "MY_NODE_NAME"
  "valueFrom":
  "fieldRef":

config/master/manifests.jsonnet

@@ -40,9 +40,14 @@ local awsnode = {
  },
  {
  apiGroups: [""],
- resources: ["pods", "nodes", "namespaces"],
+ resources: ["pods", "namespaces"],
  verbs: ["list", "watch", "get"],
  },
+ {
+ apiGroups: [""],
+ resources: ["nodes"],
+ verbs: ["list", "watch", "get", "update"],
+ },
  {
  apiGroups: ["extensions"],
  resources: ["*"],
@@ -155,26 +160,27 @@ local awsnode = {
  initialDelaySeconds: 60,
  },
  env_:: {
+ ADDITIONAL_ENI_TAGS: "{}",
  AWS_VPC_CNI_NODE_PORT_SUPPORT: "true",
- AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG: "false",
  AWS_VPC_ENI_MTU: "9001",
+ AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER: "false",
+ AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG: "false",
  AWS_VPC_K8S_CNI_EXTERNALSNAT: "false",
- AWS_VPC_K8S_CNI_RANDOMIZESNAT: "prng",
- WARM_ENI_TARGET: "1",
  AWS_VPC_K8S_CNI_LOGLEVEL: "DEBUG",
  AWS_VPC_K8S_CNI_LOG_FILE: "/host/var/log/aws-routed-eni/ipamd.log",
+ AWS_VPC_K8S_CNI_RANDOMIZESNAT: "prng",
+ AWS_VPC_K8S_CNI_VETHPREFIX: "eni",
  AWS_VPC_K8S_PLUGIN_LOG_FILE: "/var/log/aws-routed-eni/plugin.log",
  AWS_VPC_K8S_PLUGIN_LOG_LEVEL: "DEBUG",
  DISABLE_INTROSPECTION: "false",
  DISABLE_METRICS: "false",
- AWS_VPC_K8S_CNI_VETHPREFIX: "eni",
- ADDITIONAL_ENI_TAGS: "{}",
- AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER: "false",
+ ENABLE_POD_ENI: "false",
  MY_NODE_NAME: {
  valueFrom: {
  fieldRef: {fieldPath: "spec.nodeName"},
  },
  },
+ WARM_ENI_TARGET: "1",
  },
  env: [
  {name: kv[0]} + if std.isObject(kv[1]) then kv[1] else {value: kv[1]}

pkg/awsutils/awsutils.go

@@ -126,7 +126,7 @@ type APIs interface {
  GetIPv4sFromEC2(eniID string) (addrList []*ec2.NetworkInterfacePrivateIpAddress, err error)
 
  // DescribeAllENIs calls EC2 and returns the ENIMetadata and a tag map for each ENI
- DescribeAllENIs() ([]ENIMetadata, map[string]TagMap, error)
+ DescribeAllENIs() (eniMetadata []ENIMetadata, tagMap map[string]TagMap, trunkENI string, err error)
 
  // AllocIPAddress allocates an IP address for an ENI
  AllocIPAddress(eniID string) error
@@ -729,6 +729,7 @@ func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string
  } else {
  log.Info("Using same config as the primary interface for the new ENI")
  }
+
  var sgs []string
  for i := range input.Groups {
  sgs = append(sgs, *input.Groups[i])
@@ -998,11 +999,11 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [
 }
 
 // DescribeAllENIs calls EC2 to refrech the ENIMetadata and tags for all attached ENIs
-func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[string]TagMap, error) {
+func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (eniMetadata []ENIMetadata, tagMap map[string]TagMap, trunkENI string, err error) {
  // Fetch all local ENI info from metadata
  allENIs, err := cache.GetAttachedENIs()
  if err != nil {
- return nil, nil, errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata")
+ return nil, nil, "", errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata")
  }
 
  eniMap := make(map[string]ENIMetadata, len(allENIs))
@@ -1047,7 +1048,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
  }
 
  if err != nil {
- return nil, nil, err
+ return nil, nil, "", err
  }
 
  // Collect the verified ENIs
@@ -1057,13 +1058,18 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
  }
 
  // Collect ENI response into ENI metadata and tags.
- tagMap := make(map[string]TagMap, len(ec2Response.NetworkInterfaces))
+ tagMap = make(map[string]TagMap, len(ec2Response.NetworkInterfaces))
  for _, ec2res := range ec2Response.NetworkInterfaces {
  if ec2res.Attachment != nil && aws.Int64Value(ec2res.Attachment.DeviceIndex) == 0 && !aws.BoolValue(ec2res.Attachment.DeleteOnTermination) {
  log.Warn("Primary ENI will not get deleted when node terminates because 'delete_on_termination' is set to false")
  }
  eniID := aws.StringValue(ec2res.NetworkInterfaceId)
  eniMetadata := eniMap[eniID]
+ interfaceType := aws.StringValue(ec2res.InterfaceType)
+ // This assumes we only have one trunk attached to the node..
+ if interfaceType == "trunk" {
+ trunkENI = eniID
+ }
  // Check IPv4 addresses
  logOutOfSyncState(eniID, eniMetadata.IPv4Addresses, ec2res.PrivateIpAddresses)
  tags := make(map[string]string, len(ec2res.TagSet))
@@ -1078,7 +1084,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
  tagMap[eniMetadata.ENIID] = tags
  }
  }
- return verifiedENIs, tagMap, nil
+ return verifiedENIs, tagMap, trunkENI, nil
 }
 
 var eniErrorMessageRegex = regexp.MustCompile("'([a-zA-Z0-9-]+)'")

pkg/awsutils/awsutils_test.go

@@ -393,7 +393,7 @@ func TestDescribeAllENIs(t *testing.T) {
  for _, tc := range testCases {
  mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(tc.n).Return(result, tc.awsErr)
  ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata, ec2SVC: mockEC2}
- _, tags, err := ins.DescribeAllENIs()
+ _, tags, _, err := ins.DescribeAllENIs()
  assert.Equal(t, tc.expErr, err, tc.name)
  assert.Equal(t, tc.exptags, tags, tc.name)
  }

pkg/eniconfig/eniconfig.go

@@ -126,12 +126,17 @@ func (h *Handler) Handle(ctx context.Context, event sdk.Event) error {
  val = eniConfigDefault
  }
  }
-
+ // If value changes
  if h.controller.myENI != val {
  h.controller.eniLock.Lock()
  defer h.controller.eniLock.Unlock()
  h.controller.myENI = val
  log.Debugf("Setting myENI to: %s", val)
+ if val != eniConfigDefault {
+ labels := o.GetLabels()
+ labels["vpc.amazonaws.com/eniConfig"] = val
+ o.SetLabels(labels)
+ }
  }
  }
  }

pkg/eniconfig/eniconfig_test.go

@@ -46,7 +46,8 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string,
  node := corev1.Node{
  TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
  ObjectMeta: metav1.ObjectMeta{
- Name: nodeName,
+ Name: nodeName,
+ Labels: make(map[string]string),
  },
  }
  accessor, err := meta.Accessor(&node)

pkg/ipamd/datastore/data_store.go

@@ -115,13 +115,15 @@ func (k IPAMKey) String() string {
  return fmt.Sprintf("%s/%s/%s", k.NetworkName, k.ContainerID, k.IfName)
 }
 
-// ENI represents a single ENI.
+// ENI represents a single ENI. Exported fields will be marshaled for introspection.
 type ENI struct {
  // AWS ENI ID
  ID string
  createTime time.Time
  // IsPrimary indicates whether ENI is a primary ENI
  IsPrimary bool
+ // IsTrunk indicates whether this ENI is used to provide pods with dedicated ENIs
+ IsTrunk bool
  // DeviceNumber is the device number of ENI (0 means the primary ENI)
  DeviceNumber int
  // IPv4Addresses shows whether each address is assigned, the key is IP address, which must
@@ -355,7 +357,7 @@ func (ds *DataStore) writeBackingStoreUnsafe() error {
 }
 
 // AddENI add ENI to data store
-func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) error {
+func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk bool) error {
  ds.lock.Lock()
  defer ds.lock.Unlock()
 
@@ -368,6 +370,7 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) erro
  ds.eniPool[eniID] = &ENI{
  createTime: time.Now(),
  IsPrimary: isPrimary,
+ IsTrunk: isTrunk,
  ID: eniID,
  DeviceNumber: deviceNumber,
  IPv4Addresses: make(map[string]*AddressInfo)}
@@ -505,6 +508,17 @@ func (ds *DataStore) GetStats() (int, int) {
  return ds.total, ds.assigned
 }
 
+func (ds *DataStore) GetTrunkENI() string {
+ ds.lock.Lock()
+ defer ds.lock.Unlock()
+ for _, eni := range ds.eniPool {
+ if eni.IsTrunk {
+ return eni.ID
+ }
+ }
+ return ""
+}
+
 // IsRequiredForWarmIPTarget determines if this ENI has warm IPs that are required to fulfill whatever WARM_IP_TARGET is
 // set to.
 func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool {
@@ -561,6 +575,11 @@ func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI
  continue
  }
 
+ if eni.IsTrunk {
+ ds.log.Debugf("ENI %s cannot be deleted because it is a trunk ENI", eni.ID)
+ continue
+ }
+
  ds.log.Debugf("getDeletableENI: found a deletable ENI %s", eni.ID)
  return eni
  }