From d6a1ceec4b521e5a27d9159f7744209f4776e274 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Thu, 21 Oct 2021 01:01:45 -0700 Subject: [PATCH] # This is a combination of 11 commits. # This is the 1st commit message: Add VlanId in the cmdAdd Result struct This VlanId will appear in the prevResult during cmdDel request Test prevResult contents CleanUp Pod Network using vlanId from prevResult in CNI itself No need to call ipamd Log formatting changes Added hostNetworking Setup test for pods using security groups revoke unnecessary test agent image changes Revoke unnecessary changes remove focussed test set replica count to total number of branch interface Fix replica count # This is the commit message #2: Updated cleanUpPodENI method # This is the commit message #3: Skip processing Delete request if prevResult is nil Add Logging vlanId to ipamd # This is the commit message #4: Add support to test with containerd nodegroup in pod-eni test # This is the commit message #5: Add check for empty Netns() in cni # This is the commit message #6: Manifests and Readme updates (#1732) * Manifests and Readme updates * update manifest.jsonnet # This is the commit message #7: Readme updates (#1735) # This is the commit message #8: Updates to troubleshooting doc (#1737) * Updates to troubleshooting doc * updates to troubleshooting doc # This is the commit message #9: imdsv2 changes (#1743) # This is the commit message #10: fix flaky canary test (#1742) # This is the commit message #11: add CODEOWNERS (#1747) --- CODEOWNERS | 1 + README.md | 79 ++++++++++-- charts/aws-vpc-cni/Chart.yaml | 4 +- charts/aws-vpc-cni/values.yaml | 5 +- charts/cni-metrics-helper/Chart.yaml | 4 +- charts/cni-metrics-helper/values.yaml | 2 +- cmd/routed-eni-cni-plugin/cni.go | 56 ++++++++- config/master/aws-k8s-cni-cn.yaml | 16 +-- config/master/aws-k8s-cni-us-gov-east-1.yaml | 16 +-- config/master/aws-k8s-cni-us-gov-west-1.yaml | 16 +-- config/master/aws-k8s-cni.yaml | 16 +-- config/master/cni-metrics-helper-cn.yaml | 6 +- .../cni-metrics-helper-us-gov-east-1.yaml | 6 +- .../cni-metrics-helper-us-gov-west-1.yaml | 6 +- config/master/cni-metrics-helper.yaml | 6 +- config/master/manifests.jsonnet | 2 +- docs/troubleshooting.md | 7 ++ misc/10-aws.conflist | 2 +- pkg/ipamd/rpc_handler.go | 2 + scripts/entrypoint.sh | 5 +- .../securiy_group_per_pod_suite_test.go | 7 +- .../e2e/pod-eni/securiy_group_per_pod_test.go | 114 +++++++++++++++++- .../resources/aws/utils/nodegroup.go | 7 ++ .../resources/k8s/resources/service.go | 16 ++- .../resources/k8s/utils/daemonset.go | 2 +- .../cni/service_connectivity_test.go | 13 +- .../ipv6/ipv6_service_connectivity_test.go | 2 +- 27 files changed, 344 insertions(+), 74 deletions(-) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000000..9228aad0315 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @aws/eks-networking diff --git a/README.md b/README.md index 9a665cad767..aafd65b46f4 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,11 @@ scheduling that exceeds the IP address resources available to the kubelet. The default manifest expects `--cni-conf-dir=/etc/cni/net.d` and `--cni-bin-dir=/opt/cni/bin`. -L-IPAM requires following [IAM policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html): +## IAM Policy + +L-IPAM requires one of the following [IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) depending on the IP Family configured: + +**IPv4 Mode:** ``` { @@ -56,6 +60,31 @@ L-IPAM requires following [IAM policy](https://docs.aws.amazon.com/IAM/latest/Us } ``` +**IPv6 Mode:** + +``` + { + "Effect": "Allow", + "Action": [ + "ec2:AssignIpv6Addresses", + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeInstanceTypes" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": [ + "arn:aws:ec2:*:*:network-interface/*" + ] + } +``` + Alternatively there is also a [Helm](https://helm.sh/) chart: [eks/aws-vpc-cni](https://github.com/aws/eks-charts/tree/master/stable/aws-vpc-cni) ## Building @@ -474,14 +503,16 @@ Type: Boolean as a String Default: `false` -To enable IPv4 prefix delegation on nitro instances. Setting `ENABLE_PREFIX_DELEGATION` to `true` will start allocating a /28 prefix -instead of a secondary IP in the ENIs subnet. The total number of prefixes and private IP addresses will be less than the +To enable prefix delegation on nitro instances. Setting `ENABLE_PREFIX_DELEGATION` to `true` will start allocating a prefix (/28 for IPv4 +and /80 for IPv6) instead of a secondary IP in the ENIs subnet. The total number of prefixes and private IP addresses will be less than the limit on private IPs allowed by your instance. Setting or resetting of `ENABLE_PREFIX_DELEGATION` while pods are running or if ENIs are attached is supported and the new pods allocated will get IPs based on the mode of IPAMD but the max pods of kubelet should be updated which would need either kubelet restart or node recycle. -Custom networking and Security group per pods are supported with this feature. - Setting ENABLE_PREFIX_DELEGATION to true will not increase the density of branch ENI pods. The limit on number of branch network interfaces per instance type will remain the same - https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types. Each branch network will be allocated a primary IP and this IP will be allocated for the branch ENI pods. +Please refer to [VPC CNI Feature Matrix](https://github.com/aws/amazon-vpc-cni-k8s#vpc-cni-feature-matrix) section below for additional information around using Prefix delegation with Custom Networking and Security Groups Per Pod features. + +**Note:** `ENABLE_PREFIX_DELEGATION` needs to be set to `true` when VPC CNI is configured to operate in IPv6 mode (supported in v1.10.0+). + --- #### `WARM_PREFIX_TARGET` (v1.9.0+) @@ -522,10 +553,10 @@ Type: Boolean as a String Default: `false` Setting `ANNOTATE_POD_IP` to `true` will allow IPAMD to add an annotation `vpc.amazonaws.com/pod-ips` to the pod with pod IP. - + There is a known [issue](https://github.com/kubernetes/kubernetes/issues/39113) with kubelet taking time to update `Pod.Status.PodIP` leading to calico being blocked on programming the policy. Setting `ANNOTATE_POD_IP` to `true` will enable AWS VPC CNI plugin to add Pod IP as an annotation to the pod spec to address this race condition. -To annotate the pod with pod IP, you will have to add "patch" permission for pods resource in aws-node clusterrole. You can use the below command - +To annotate the pod with pod IP, you will have to add "patch" permission for pods resource in aws-node clusterrole. You can use the below command - ``` cat << EOF > append.yaml @@ -543,6 +574,40 @@ kubectl apply -f <(cat <(kubectl get clusterrole aws-node -o yaml) append.yaml) ``` --- +#### `ENABLE_IPv4` (v1.10.0+) + +Type: Boolean as a String + +Default: `true` + +VPC CNI can operate in either IPv4 or IPv6 mode. Setting `ENABLE_IPv4` to `true` will configure it in IPv4 mode (default mode). + +**Note:** Dual stack mode isn't yet supported. So, enabling both IPv4 and IPv6 will be treated as invalid configuration. + +--- + +#### `ENABLE_IPv6` (v1.10.0+) + +Type: Boolean as a String + +Default: `false` + +VPC CNI can operate in either IPv4 or IPv6 mode. Setting `ENABLE_IPv6` to `true` (both under `aws-node` and `aws-vpc-cni-init` containers in the manifest) +will configure it in IPv6 mode. IPv6 is only supported in Prefix Delegation mode, so `ENABLE_PREFIX_DELEGATION` needs to set to `true` if VPC CNI is +configured to operate in IPv6 mode. Prefix delegation is only supported on nitro instances. + + +**Note:** Please make sure that the required IPv6 IAM policy is applied (Refer to [IAM Policy](https://github.com/aws/amazon-vpc-cni-k8s#iam-policy) section above). Dual stack mode isn't yet supported. So, enabling both IPv4 and IPv6 will be treated as invalid configuration. Please refer to the [VPC CNI Feature Matrix](https://github.com/aws/amazon-vpc-cni-k8s#vpc-cni-feature-matrix) section below for additional information. + +--- + +### VPC CNI Feature Matrix + +IP Mode | Secondary IP Mode | Prefix Delegation | Security Groups Per Pod | WARM & MIN IP/Prefix Targets | External SNAT +------ | ------ | ------ | ------ | ------ | ------ +`IPv4` | Yes| Yes | Yes | Yes | Yes | Yes +`IPv6` | No | Yes | No | No | No | No + ### ENI tags related to Allocation This plugin interacts with the following tags on ENIs: diff --git a/charts/aws-vpc-cni/Chart.yaml b/charts/aws-vpc-cni/Chart.yaml index 74d24d0cc16..637118739fa 100644 --- a/charts/aws-vpc-cni/Chart.yaml +++ b/charts/aws-vpc-cni/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: aws-vpc-cni -version: 1.1.10 -appVersion: "v1.9.3" +version: 1.1.11 +appVersion: "v1.10.0" description: A Helm chart for the AWS VPC CNI icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png home: https://github.com/aws/amazon-vpc-cni-k8s diff --git a/charts/aws-vpc-cni/values.yaml b/charts/aws-vpc-cni/values.yaml index 088c40f8597..95ef9c6aa08 100644 --- a/charts/aws-vpc-cni/values.yaml +++ b/charts/aws-vpc-cni/values.yaml @@ -8,7 +8,7 @@ nameOverride: aws-node init: image: - tag: v1.9.3 + tag: v1.10.0 region: us-west-2 account: "602401143452" pullPolicy: Always @@ -17,12 +17,13 @@ init: # override: "repo/org/image:tag" env: DISABLE_TCP_EARLY_DEMUX: "false" + ENABLE_IPv6: "false" securityContext: privileged: true image: region: us-west-2 - tag: v1.9.3 + tag: v1.10.0 account: "602401143452" domain: "amazonaws.com" pullPolicy: Always diff --git a/charts/cni-metrics-helper/Chart.yaml b/charts/cni-metrics-helper/Chart.yaml index bea5e6515e3..8d9f0506e27 100644 --- a/charts/cni-metrics-helper/Chart.yaml +++ b/charts/cni-metrics-helper/Chart.yaml @@ -15,9 +15,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.4 +version: 0.1.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: v1.9.3 +appVersion: v1.10.0 diff --git a/charts/cni-metrics-helper/values.yaml b/charts/cni-metrics-helper/values.yaml index 4d6f8e99109..33764fbe9f9 100644 --- a/charts/cni-metrics-helper/values.yaml +++ b/charts/cni-metrics-helper/values.yaml @@ -4,7 +4,7 @@ nameOverride: cni-metrics-helper image: region: us-west-2 - tag: v1.9.3 + tag: v1.10.0 account: "602401143452" domain: "amazonaws.com" # Set to use custom image diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index 7a799ea4f03..9cdf7c5b70a 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -22,6 +22,7 @@ import ( "net" "os" "runtime" + "strconv" "strings" "github.com/containernetworking/cni/pkg/skel" @@ -43,6 +44,7 @@ import ( ) const ipamdAddress = "127.0.0.1:50051" +const vlanInterfaceName = "vlanId" var version string @@ -95,6 +97,12 @@ func LoadNetConf(bytes []byte) (*NetConf, logger.Logger, error) { return nil, nil, errors.Wrap(err, "add cmd: error loading config from args") } + if conf.RawPrevResult != nil { + if err := cniSpecVersion.ParsePrevResult(&conf.NetConf); err != nil { + return nil, nil, fmt.Errorf("could not parse prevResult: %v", err) + } + } + logConfig := logger.Configuration{ LogLevel: conf.PluginLogLevel, LogLocation: conf.PluginLogFile, @@ -122,6 +130,8 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap log.Infof("Received CNI add request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) + log.Infof("Prev Result: %v\n", conf.PrevResult) + var k8sArgs K8sArgs if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil { log.Errorf("Failed to load k8s config from arg: %v", err) @@ -194,14 +204,12 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap var hostVethName string if r.PodVlanId != 0 { hostVethName = generateHostVethName("vlan", string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) - err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC, r.PodENISubnetGW, int(r.ParentIfIndex), mtu, log) } else { // build hostVethName // Note: the maximum length for linux interface name is 15 hostVethName = generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) - err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), r.VPCv4CIDRs, r.UseExternalSNAT, mtu, log) } @@ -241,12 +249,15 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap hostInterface := ¤t.Interface{Name: hostVethName} containerInterface := ¤t.Interface{Name: args.IfName, Sandbox: args.Netns} + vlanInterface := ¤t.Interface{Name: vlanInterfaceName, Mac: fmt.Sprint(r.PodVlanId)} + log.Infof("Using vlanInterface: %v", vlanInterface) result := ¤t.Result{ IPs: ips, Interfaces: []*current.Interface{ hostInterface, containerInterface, + vlanInterface, }, } @@ -270,6 +281,8 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap driverClient driver.NetworkAPIs) error { conf, log, err := LoadNetConf(args.StdinData) + log.Infof("Prev Result: %v\n", conf.PrevResult) + if err != nil { return errors.Wrap(err, "add cmd: error loading config from args") } @@ -283,6 +296,35 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap return errors.Wrap(err, "del cmd: failed to load k8s config from args") } + prevResult, ok := conf.PrevResult.(*current.Result) + + if !ok || args.Netns == "" { + log.Info("prevResult is nil or Netns() is empty, skip processing this request") + return nil + } + + for _, iface := range prevResult.Interfaces { + if iface.Name == vlanInterfaceName { + podVlanId, err := strconv.Atoi(iface.Mac) + if err != nil { + return errors.Wrap(err, "Failed to parse vlanId from prevResult") + } + // podVlanId == 0 means pod is not using branch ENI + // then fallback to existing cleanup + if podVlanId == 0 { + break + } + // if podVlanId != 0 means pod is using branch ENI + err = cleanUpPodENI(podVlanId, log, args.ContainerID, driverClient) + if err != nil { + return err + } + log.Infof("Received del network response for pod %s namespace %s sandbox %s with vlanId: %v", string(k8sArgs.K8S_POD_NAME), + string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), podVlanId) + return nil + } + } + // notify local IP address manager to free secondary IP // Set up a connection to the server. conn, err := grpcClient.Dial(ipamdAddress, grpc.WithInsecure()) @@ -362,6 +404,16 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap return nil } +func cleanUpPodENI(podVlanId int, log logger.Logger, containerId string, driverClient driver.NetworkAPIs) error { + err := driverClient.TeardownPodENINetwork(podVlanId, log) + if err != nil { + log.Errorf("Failed on TeardownPodNetwork for container ID %s: %v", + containerId, err) + return errors.Wrap(err, "del cmd: failed on tear down pod network") + } + return nil +} + func main() { log := logger.DefaultLogger() about := fmt.Sprintf("AWS CNI %s", version) diff --git a/config/master/aws-k8s-cni-cn.yaml b/config/master/aws-k8s-cni-cn.yaml index 2e1f32b260c..9d0fe6ca89b 100644 --- a/config/master/aws-k8s-cni-cn.yaml +++ b/config/master/aws-k8s-cni-cn.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: aws-vpc-cni/templates/customresourcedefinition.yaml apiVersion: apiextensions.k8s.io/v1 @@ -20,7 +20,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: scope: Cluster group: crd.k8s.amazonaws.com @@ -47,7 +47,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" rules: - apiGroups: - crd.k8s.amazonaws.com @@ -80,7 +80,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -100,7 +100,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: updateStrategy: rollingUpdate: @@ -121,10 +121,12 @@ spec: hostNetwork: true initContainers: - name: aws-vpc-cni-init - image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.9.3" + image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.10.0" env: - name: DISABLE_TCP_EARLY_DEMUX value: "false" + - name: ENABLE_IPv6 + value: "false" securityContext: privileged: true volumeMounts: @@ -137,7 +139,7 @@ spec: {} containers: - name: aws-node - image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.9.3" + image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.10.0" ports: - containerPort: 61678 name: metrics diff --git a/config/master/aws-k8s-cni-us-gov-east-1.yaml b/config/master/aws-k8s-cni-us-gov-east-1.yaml index 1e2cb79d0a2..7054b0d87f3 100644 --- a/config/master/aws-k8s-cni-us-gov-east-1.yaml +++ b/config/master/aws-k8s-cni-us-gov-east-1.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: aws-vpc-cni/templates/customresourcedefinition.yaml apiVersion: apiextensions.k8s.io/v1 @@ -20,7 +20,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: scope: Cluster group: crd.k8s.amazonaws.com @@ -47,7 +47,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" rules: - apiGroups: - crd.k8s.amazonaws.com @@ -80,7 +80,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -100,7 +100,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: updateStrategy: rollingUpdate: @@ -121,10 +121,12 @@ spec: hostNetwork: true initContainers: - name: aws-vpc-cni-init - image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni-init:v1.9.3" + image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni-init:v1.10.0" env: - name: DISABLE_TCP_EARLY_DEMUX value: "false" + - name: ENABLE_IPv6 + value: "false" securityContext: privileged: true volumeMounts: @@ -137,7 +139,7 @@ spec: {} containers: - name: aws-node - image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni:v1.9.3" + image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni:v1.10.0" ports: - containerPort: 61678 name: metrics diff --git a/config/master/aws-k8s-cni-us-gov-west-1.yaml b/config/master/aws-k8s-cni-us-gov-west-1.yaml index fc2e2984168..850f9b86872 100644 --- a/config/master/aws-k8s-cni-us-gov-west-1.yaml +++ b/config/master/aws-k8s-cni-us-gov-west-1.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: aws-vpc-cni/templates/customresourcedefinition.yaml apiVersion: apiextensions.k8s.io/v1 @@ -20,7 +20,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: scope: Cluster group: crd.k8s.amazonaws.com @@ -47,7 +47,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" rules: - apiGroups: - crd.k8s.amazonaws.com @@ -80,7 +80,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -100,7 +100,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: updateStrategy: rollingUpdate: @@ -121,10 +121,12 @@ spec: hostNetwork: true initContainers: - name: aws-vpc-cni-init - image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni-init:v1.9.3" + image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni-init:v1.10.0" env: - name: DISABLE_TCP_EARLY_DEMUX value: "false" + - name: ENABLE_IPv6 + value: "false" securityContext: privileged: true volumeMounts: @@ -137,7 +139,7 @@ spec: {} containers: - name: aws-node - image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni:v1.9.3" + image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/amazon-k8s-cni:v1.10.0" ports: - containerPort: 61678 name: metrics diff --git a/config/master/aws-k8s-cni.yaml b/config/master/aws-k8s-cni.yaml index 4e7848f761f..95ea6d532f8 100644 --- a/config/master/aws-k8s-cni.yaml +++ b/config/master/aws-k8s-cni.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: aws-vpc-cni/templates/customresourcedefinition.yaml apiVersion: apiextensions.k8s.io/v1 @@ -20,7 +20,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: scope: Cluster group: crd.k8s.amazonaws.com @@ -47,7 +47,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" rules: - apiGroups: - crd.k8s.amazonaws.com @@ -80,7 +80,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -100,7 +100,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" spec: updateStrategy: rollingUpdate: @@ -121,10 +121,12 @@ spec: hostNetwork: true initContainers: - name: aws-vpc-cni-init - image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.9.3" + image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.10.0" env: - name: DISABLE_TCP_EARLY_DEMUX value: "false" + - name: ENABLE_IPv6 + value: "false" securityContext: privileged: true volumeMounts: @@ -137,7 +139,7 @@ spec: {} containers: - name: aws-node - image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.9.3" + image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.10.0" ports: - containerPort: 61678 name: metrics diff --git a/config/master/cni-metrics-helper-cn.yaml b/config/master/cni-metrics-helper-cn.yaml index 5435b66e523..86ef3ee8b01 100644 --- a/config/master/cni-metrics-helper-cn.yaml +++ b/config/master/cni-metrics-helper-cn.yaml @@ -8,7 +8,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: cni-metrics-helper/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -58,7 +58,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -90,5 +90,5 @@ spec: - name: USE_CLOUDWATCH value: "true" name: cni-metrics-helper - image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/cni-metrics-helper:v1.9.3" + image: "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/cni-metrics-helper:v1.10.0" serviceAccountName: cni-metrics-helper diff --git a/config/master/cni-metrics-helper-us-gov-east-1.yaml b/config/master/cni-metrics-helper-us-gov-east-1.yaml index 85a9764cee3..16412e7fd1e 100644 --- a/config/master/cni-metrics-helper-us-gov-east-1.yaml +++ b/config/master/cni-metrics-helper-us-gov-east-1.yaml @@ -8,7 +8,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: cni-metrics-helper/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -58,7 +58,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -90,5 +90,5 @@ spec: - name: USE_CLOUDWATCH value: "true" name: cni-metrics-helper - image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/cni-metrics-helper:v1.9.3" + image: "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/cni-metrics-helper:v1.10.0" serviceAccountName: cni-metrics-helper diff --git a/config/master/cni-metrics-helper-us-gov-west-1.yaml b/config/master/cni-metrics-helper-us-gov-west-1.yaml index 9bdcf85f2bf..0c6b6a2aaeb 100644 --- a/config/master/cni-metrics-helper-us-gov-west-1.yaml +++ b/config/master/cni-metrics-helper-us-gov-west-1.yaml @@ -8,7 +8,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: cni-metrics-helper/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -58,7 +58,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -90,5 +90,5 @@ spec: - name: USE_CLOUDWATCH value: "true" name: cni-metrics-helper - image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/cni-metrics-helper:v1.9.3" + image: "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/cni-metrics-helper:v1.10.0" serviceAccountName: cni-metrics-helper diff --git a/config/master/cni-metrics-helper.yaml b/config/master/cni-metrics-helper.yaml index 141232320c4..0e2381ffb3c 100644 --- a/config/master/cni-metrics-helper.yaml +++ b/config/master/cni-metrics-helper.yaml @@ -8,7 +8,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" --- # Source: cni-metrics-helper/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -58,7 +58,7 @@ metadata: labels: app.kubernetes.io/name: cni-metrics-helper app.kubernetes.io/instance: cni-metrics-helper - app.kubernetes.io/version: "v1.9.3" + app.kubernetes.io/version: "v1.10.0" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -90,5 +90,5 @@ spec: - name: USE_CLOUDWATCH value: "true" name: cni-metrics-helper - image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper:v1.9.3" + image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper:v1.10.0" serviceAccountName: cni-metrics-helper diff --git a/config/master/manifests.jsonnet b/config/master/manifests.jsonnet index f7af72f670a..2eb73cdeab3 100644 --- a/config/master/manifests.jsonnet +++ b/config/master/manifests.jsonnet @@ -3,7 +3,7 @@ local objectItems(obj) = [[k, obj[k]] for k in std.objectFields(obj)]; local regions = { default: { - version:: "v1.9.3", // or eg "v1.6.2" + version:: "v1.10.0", // or eg "v1.6.2" ecrRegion:: "us-west-2", ecrAccount:: "602401143452", ecrDomain:: "amazonaws.com", diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d5bdf1ff7a1..a2d4a7a67f2 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -191,6 +191,13 @@ go_goroutines 20 ... ``` +## IMDS + +If you're using v1.10.0, `aws-node` daemonset pod requires IMDSv1 access to obtain Primary IPv4 address assigned to the Node. Please refer to `Block access to IMDSv1 and IMDSv2 for all containers that don't use host networking` section in this [doc](https://docs.aws.amazon.com/eks/latest/userguide/best-practices-security.html) + +## Known Issues +- **Liveness/Readiness Probe failures** - If frequent probe failures are observed for `aws-node` pods in v1.20+ clusters, please bump up the liveness/readiness probe timeout values and/or CPU requests/limts in the CNI Manifest. Refer to this github [issue](https://github.com/aws/amazon-vpc-cni-k8s/issues/1425) + ## cni-metrics-helper See the [cni-metrics-helper README](../cmd/cni-metrics-helper/README.md). diff --git a/misc/10-aws.conflist b/misc/10-aws.conflist index fb52e797f49..ff2c5319190 100644 --- a/misc/10-aws.conflist +++ b/misc/10-aws.conflist @@ -1,5 +1,5 @@ { - "cniVersion": "0.3.1", + "cniVersion": "0.4.0", "name": "aws-cni", "plugins": [ { diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index 962767363c8..35898fe2f6e 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -110,6 +110,8 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp ipv4Addr = firstENI.PrivateIP branchENIMAC = firstENI.IfAddress vlanID = firstENI.VlanID + log.Infof("Pod vlandId: %d", vlanID) + if ipv4Addr == "" || branchENIMAC == "" || vlanID == 0 { log.Errorf("Failed to parse pod-ENI annotation: %s", val) return &failureResponse, nil diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 6eacd367ea1..edaf78642a7 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -118,6 +118,7 @@ wait_for_ipam() { fi # We sleep for 1 second between each retry sleep 1 + log_in_json info "Retrying waiting for IPAM-D" done } @@ -125,12 +126,14 @@ wait_for_ipam() { get_node_primary_v4_address() { while : do - NODE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) + token=$(curl -Ss -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") + NODE_IP=$(curl -H "X-aws-ec2-metadata-token: $token" -Ss http://169.254.169.254/latest/meta-data/local-ipv4) if [[ "${NODE_IP}" != "" ]]; then return 0 fi # We sleep for 1 second between each retry sleep 1 + log_in_json info "Retrying fetching node-IP" done } diff --git a/test/e2e/pod-eni/securiy_group_per_pod_suite_test.go b/test/e2e/pod-eni/securiy_group_per_pod_suite_test.go index b5408f2ae74..afc7250d283 100644 --- a/test/e2e/pod-eni/securiy_group_per_pod_suite_test.go +++ b/test/e2e/pod-eni/securiy_group_per_pod_suite_test.go @@ -23,8 +23,9 @@ import ( awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" - "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" + v1 "k8s.io/api/core/v1" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -52,6 +53,8 @@ var ( clusterRoleName string // NodeSecurityGroupId for Node-Node communication nodeSecurityGroupID string + + node v1.Node ) func TestSecurityGroupForPods(t *testing.T) { @@ -116,7 +119,7 @@ var _ = BeforeSuite(func() { // Get ref to any node from newly created nodegroup By("Getting providerID of the node") - node := nodeList.Items[0] + node = nodeList.Items[0] providerID := node.Spec.ProviderID Expect(len(providerID)).To(BeNumerically(">", 0)) diff --git a/test/e2e/pod-eni/securiy_group_per_pod_test.go b/test/e2e/pod-eni/securiy_group_per_pod_test.go index 121acbbafcd..f522078b7ca 100644 --- a/test/e2e/pod-eni/securiy_group_per_pod_test.go +++ b/test/e2e/pod-eni/securiy_group_per_pod_test.go @@ -18,6 +18,7 @@ import ( "fmt" "time" + "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/agent" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" @@ -32,13 +33,21 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +type TestType int + +const ( + NetworkingTearDownSucceeds TestType = iota + NetworkingSetupSucceeds +) + var _ = Describe("Security Group for Pods Test", func() { var ( // The Pod labels for client and server in order to retrieve the // client and server Pods belonging to a Deployment/Jobs - labelKey = "app" - serverPodLabelVal = "server-pod" - clientPodLabelVal = "client-pod" + labelKey = "app" + serverPodLabelVal = "server-pod" + clientPodLabelVal = "client-pod" + busyboxPodLabelVal = "busybox-pod" // The Security Group Policy take list of Pod Label Value and if the // Pod has any label in the list, it should get Branch ENI branchPodLabelVal []string @@ -264,8 +273,107 @@ var _ = Describe("Security Group for Pods Test", func() { It("TCP liveness probe will succeed", func() {}) }) }) + + Context("Verify HostNetworking", func() { + BeforeEach(func() { + // BusyBox Pods will get Branch ENI + branchPodLabelVal = []string{busyboxPodLabelVal} + }) + It("Deploy BusyBox Pods with branch ENI and verify HostNetworking", func() { + deployment := manifest.NewBusyBoxDeploymentBuilder(). + Replicas(totalBranchInterface/asgSize). + PodLabel(labelKey, busyboxPodLabelVal). + NodeName(node.Name). + Build() + + By("creating a deployment to launch pod using Branch ENI") + _, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("getting the list of pods using BranchENI") + podList, err := f.K8sResourceManagers. + PodManager(). + GetPodsWithLabelSelector(labelKey, busyboxPodLabelVal) + Expect(err).ToNot(HaveOccurred()) + + By("generating the pod networking validation input to be passed to tester") + input, err := GetPodNetworkingValidationInput(podList).Serialize() + Expect(err).NotTo(HaveOccurred()) + + By("validating host networking setup is setup correctly") + ValidateHostNetworking(NetworkingSetupSucceeds, input) + + By("deleting the deployment to test teardown") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("waiting to allow CNI to tear down networking for terminated pods") + time.Sleep(time.Second * 60) + + By("validating host networking is teared down correctly") + ValidateHostNetworking(NetworkingTearDownSucceeds, input) + }) + }) }) +func GetPodNetworkingValidationInput(podList v1.PodList) input.PodNetworkingValidationInput { + ip := input.PodNetworkingValidationInput{ + VethPrefix: "vlan", + PodList: []input.Pod{}, + ValidateMTU: true, + MTU: 9001, + } + + for _, pod := range podList.Items { + ip.PodList = append(ip.PodList, input.Pod{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodIPv4Address: pod.Status.PodIP, + }) + } + return ip +} + +func ValidateHostNetworking(testType TestType, podValidationInputString string) { + testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s", + podValidationInputString)} + + if NetworkingSetupSucceeds == testType { + testerArgs = append(testerArgs, "-test-setup=true", "-test-ppsg=true") + } else if NetworkingTearDownSucceeds == testType { + testerArgs = append(testerArgs, "-test-cleanup=true", "-test-ppsg=true") + } + + testContainer := manifest.NewTestHelperContainer(). + Command([]string{"./networking"}). + Args(testerArgs). + Build() + + testPod := manifest.NewDefaultPodBuilder(). + Container(testContainer). + NodeName(node.Name). + HostNetwork(true). + Build() + + By("creating pod to test host networking setup") + testPod, err := f.K8sResourceManagers.PodManager(). + CreateAndWaitTillPodCompleted(testPod) + Expect(err).ToNot(HaveOccurred()) + + logs, errLogs := f.K8sResourceManagers.PodManager(). + PodLogs(testPod.Namespace, testPod.Name) + Expect(errLogs).ToNot(HaveOccurred()) + + fmt.Fprintln(GinkgoWriter, logs) + + By("deleting the host networking setup pod") + err = f.K8sResourceManagers.PodManager(). + DeleteAndWaitTillPodDeleted(testPod) + Expect(err).ToNot(HaveOccurred()) +} + func ValidatePodsHaveBranchENI(podList v1.PodList) error { for _, pod := range podList.Items { if val, ok := pod.Annotations["vpc.amazonaws.com/pod-eni"]; ok { diff --git a/test/framework/resources/aws/utils/nodegroup.go b/test/framework/resources/aws/utils/nodegroup.go index 47c59930b24..22fc1a3427d 100644 --- a/test/framework/resources/aws/utils/nodegroup.go +++ b/test/framework/resources/aws/utils/nodegroup.go @@ -44,6 +44,9 @@ type NodeGroupProperties struct { Subnet []string InstanceType string KeyPairName string + + // set to true for creating nodegroup with containerd runtime + EnableContainerd bool } type ClusterVPCConfig struct { @@ -94,6 +97,10 @@ func CreateAndWaitTillSelfManagedNGReady(f *framework.Framework, properties Node kubeletExtraArgs += fmt.Sprintf(" --max-pods=%d", maxPods) } + if properties.EnableContainerd { + bootstrapArgs += " --container-runtime containerd" + } + asgSizeString := strconv.Itoa(properties.AsgSize) createNgStackParams := []*cloudformation.Parameter{ diff --git a/test/framework/resources/k8s/resources/service.go b/test/framework/resources/k8s/resources/service.go index 83f34115af9..b35e12770ae 100644 --- a/test/framework/resources/k8s/resources/service.go +++ b/test/framework/resources/k8s/resources/service.go @@ -20,6 +20,7 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,7 +29,7 @@ import ( type ServiceManager interface { GetService(ctx context.Context, namespace string, name string) (*v1.Service, error) CreateService(ctx context.Context, service *v1.Service) (*v1.Service, error) - DeleteService(ctx context.Context, service *v1.Service) error + DeleteAndWaitTillServiceDeleted(ctx context.Context, service *v1.Service) error } type defaultServiceManager struct { @@ -69,11 +70,20 @@ func (s *defaultServiceManager) CreateService(ctx context.Context, service *v1.S }, ctx.Done()) } -func (s *defaultServiceManager) DeleteService(ctx context.Context, service *v1.Service) error { +func (s *defaultServiceManager) DeleteAndWaitTillServiceDeleted(ctx context.Context, service *v1.Service) error { err := s.k8sClient.Delete(ctx, service) if err != nil { return err } - return nil + observed := &v1.Service{} + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + if err := s.k8sClient.Get(ctx, utils.NamespacedName(service), observed); err != nil { + if errors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }, ctx.Done()) } diff --git a/test/framework/resources/k8s/utils/daemonset.go b/test/framework/resources/k8s/utils/daemonset.go index d5353df277a..94888bc8182 100644 --- a/test/framework/resources/k8s/utils/daemonset.go +++ b/test/framework/resources/k8s/utils/daemonset.go @@ -74,9 +74,9 @@ func updateDaemonsetEnvVarsAndWait(f *framework.Framework, dsName string, dsName // update multus daemonset if it exists // to avoid being stuck in recursive loop, we need below check if dsName != utils.MultusNodeName { - By("Restarting Multus daemonset if it exists") _, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(dsNamespace, utils.MultusNodeName) if err == nil { + By("Restarting Multus daemonset to use the update aws-node changes") td := time.Now() updateDaemonsetEnvVarsAndWait(f, utils.MultusNodeName, dsNamespace, utils.MultusContainerName, map[string]string{ "forceUpdatedAt": td.String(), diff --git a/test/integration-new/cni/service_connectivity_test.go b/test/integration-new/cni/service_connectivity_test.go index 3a7a7d923e1..94e9c944707 100644 --- a/test/integration-new/cni/service_connectivity_test.go +++ b/test/integration-new/cni/service_connectivity_test.go @@ -58,9 +58,12 @@ var _ = Describe("[CANARY] test service connectivity", func() { JustBeforeEach(func() { deploymentContainer = manifest.NewBusyBoxContainerBuilder(). - Image("python"). - Command([]string{"python3"}). - Args([]string{"-m", "http.server", "80"}).Build() + Image("nginx:1.21.4"). + Command(nil). + Port(v1.ContainerPort{ + ContainerPort: 80, + Protocol: "TCP", + }).Build() deployment = manifest.NewDefaultDeploymentBuilder(). Name("http-server"). @@ -93,7 +96,7 @@ var _ = Describe("[CANARY] test service connectivity", func() { testerContainer = manifest.NewBusyBoxContainerBuilder(). Command([]string{"wget"}). - Args([]string{"--spider", "-T", "1", fmt.Sprintf("%s:%d", service.Spec.ClusterIP, + Args([]string{"--spider", "-T", "5", fmt.Sprintf("%s:%d", service.Spec.ClusterIP, service.Spec.Ports[0].Port)}). Build() @@ -132,7 +135,7 @@ var _ = Describe("[CANARY] test service connectivity", func() { err = f.K8sResourceManagers.JobManager().DeleteAndWaitTillJobIsDeleted(negativeTesterJob) Expect(err).ToNot(HaveOccurred()) - err = f.K8sResourceManagers.ServiceManager().DeleteService(context.Background(), service) + err = f.K8sResourceManagers.ServiceManager().DeleteAndWaitTillServiceDeleted(context.Background(), service) Expect(err).ToNot(HaveOccurred()) err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) diff --git a/test/integration-new/ipv6/ipv6_service_connectivity_test.go b/test/integration-new/ipv6/ipv6_service_connectivity_test.go index d420645a0ad..54989db9d28 100644 --- a/test/integration-new/ipv6/ipv6_service_connectivity_test.go +++ b/test/integration-new/ipv6/ipv6_service_connectivity_test.go @@ -136,7 +136,7 @@ var _ = Describe("test service connectivity", func() { err = f.K8sResourceManagers.JobManager().DeleteAndWaitTillJobIsDeleted(negativeTesterJob) Expect(err).ToNot(HaveOccurred()) - err = f.K8sResourceManagers.ServiceManager().DeleteService(context.Background(), service) + err = f.K8sResourceManagers.ServiceManager().DeleteAndWaitTillServiceDeleted(context.Background(), service) Expect(err).ToNot(HaveOccurred()) err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment)