forked from aws/amazon-vpc-cni-k8s
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
cni manifest upgrade downgrade test (aws#1863)
* Added upgrade/downgrade script template Refactored code Addon upgrade/downgrade test similar to aws#1795 Added tests for addon upgrade/downgrade Changed DEFAULT version Added addon status checks Fetch latest addon version for given K8s Cluster Update kops cluster config used in weekly tests (aws#1862) * Change to kops cluster creation scripts * Add logging for retry attempt * Switch kops cluster to use docker container runtime Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Added upgrade/downgrade test for custom cni-manifest-file Added missing files remove upgrade-downgrade.sh * Add eks.go file , deleted by mistake * Extract apply manifest logic in common Remove redundant code * Add PD traffic test for cni upgrade downgrade test
- Loading branch information
Showing
6 changed files
with
339 additions
and
39 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
76 changes: 76 additions & 0 deletions
76
test/integration/cni-upgrade-downgrade/host_networking_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package cni_upgrade_downgrade | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" | ||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
| v1 "k8s.io/api/apps/v1" | ||
| ) | ||
|
|
||
| var _ = Describe("test host networking", func() { | ||
| var err error | ||
| var podLabelKey = "app" | ||
| var podLabelVal = "host-networking-test" | ||
| var deployment *v1.Deployment | ||
| var podInput string | ||
|
|
||
| Context("when pods using IP from primary and secondary ENI are created", func() { | ||
| It("should have correct host networking setup when pods are running and cleaned up when pods are terminated", func() { | ||
| By("applying initial cni manifest") | ||
| common.ApplyCNIManifest(initialManifest) | ||
|
|
||
| // Launch enough pods so some pods end up using primary ENI IP and some using secondary | ||
| // ENI IP | ||
| deployment = manifest.NewBusyBoxDeploymentBuilder(). | ||
| Replicas(maxIPPerInterface*2). | ||
| PodLabel(podLabelKey, podLabelVal). | ||
| NodeName(primaryNode.Name). | ||
| Build() | ||
|
|
||
| By("creating a deployment to launch pod using primary and secondary ENI IP") | ||
| deployment, err = f.K8sResourceManagers.DeploymentManager(). | ||
| CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| By("getting the list of pods using IP from primary and secondary ENI") | ||
| interfaceTypeToPodList := common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) | ||
|
|
||
| // Primary ENI and Secondary ENI IPs are handled differently when setting up | ||
| // the host networking rule hence this check | ||
| Expect(len(interfaceTypeToPodList.PodsOnSecondaryENI)). | ||
| Should(BeNumerically(">", 0)) | ||
| Expect(len(interfaceTypeToPodList.PodsOnPrimaryENI)). | ||
| Should(BeNumerically(">", 0)) | ||
|
|
||
| By("generating the pod networking validation input to be passed to tester") | ||
| podInput, err = common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| By("validating host networking setup is setup correctly") | ||
| common.ValidateHostNetworking(common.NetworkingSetupSucceeds, podInput, primaryNode.Name, f) | ||
|
|
||
| By("applying target cni manifest") | ||
| common.ApplyCNIManifest(targetManifest) | ||
|
|
||
| By("deleting the deployment to test teardown") | ||
| err = f.K8sResourceManagers.DeploymentManager(). | ||
| DeleteAndWaitTillDeploymentIsDeleted(deployment) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| By("waiting to allow CNI to tear down networking for terminated pods") | ||
| time.Sleep(time.Second * 60) | ||
|
|
||
| By("validating host networking is teared down correctly") | ||
| common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, podInput, primaryNode.Name, f) | ||
| }) | ||
|
|
||
| AfterEach(func() { | ||
| By("revert to initial cni manifest") | ||
| common.ApplyCNIManifest(initialManifest) | ||
| }) | ||
| }) | ||
| }) |
94 changes: 94 additions & 0 deletions
94
test/integration/cni-upgrade-downgrade/pod_traffic_PD_enabled_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"). You may | ||
| // not use this file except in compliance with the License. A copy of the | ||
| // License is located at | ||
| // | ||
| // http://aws.amazon.com/apache2.0/ | ||
| // | ||
| // or in the "license" file accompanying this file. This file is distributed | ||
| // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either | ||
| // express or implied. See the License for the specific language governing | ||
| // permissions and limitations under the License. | ||
|
|
||
| package cni_upgrade_downgrade | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" | ||
| k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" | ||
|
|
||
| . "github.com/onsi/ginkgo/v2" | ||
| ) | ||
|
|
||
| var _ = Describe("Test pod networking with prefix delegation enabled", func() { | ||
| var ( | ||
| serverDeploymentBuilder *manifest.DeploymentBuilder | ||
| // Value for the Environment variable ENABLE_PREFIX_DELEGATION | ||
| enableIPv4PrefixDelegation string | ||
| ) | ||
|
|
||
| JustBeforeEach(func() { | ||
| By("applying initial cni manifest") | ||
| common.ApplyCNIManifest(initialManifest) | ||
|
|
||
| By("creating deployment") | ||
| serverDeploymentBuilder = manifest.NewDefaultDeploymentBuilder(). | ||
| Name("traffic-server"). | ||
| NodeSelector(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) | ||
|
|
||
| By("set PD") | ||
| k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, | ||
| utils.AwsNodeNamespace, utils.AwsNodeName, | ||
| map[string]string{"ENABLE_PREFIX_DELEGATION": enableIPv4PrefixDelegation}) | ||
| }) | ||
|
|
||
| JustAfterEach(func() { | ||
| // Revert to Initial manifest file | ||
| By("revert to initial cni manifest") | ||
| common.ApplyCNIManifest(initialManifest) | ||
|
|
||
| k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, | ||
| utils.AwsNodeNamespace, utils.AwsNodeName, | ||
| map[string]string{"ENABLE_PREFIX_DELEGATION": "false"}) | ||
| }) | ||
|
|
||
| Context("when testing TCP traffic between client and server pods", func() { | ||
| BeforeEach(func() { | ||
| enableIPv4PrefixDelegation = "true" | ||
| }) | ||
|
|
||
| //TODO : Add pod IP validation if IP belongs to prefix or SIP | ||
| //TODO : remove hardcoding from client/server count | ||
| It("should have 99+% success rate", func() { | ||
| By("test with initial cni manifest file") | ||
| common.ValidateTraffic(f, serverDeploymentBuilder, 99, "tcp") | ||
| targetManifestStr := fmt.Sprintf("Testing with Target CNI Manifest: %s", targetManifest) | ||
| By(targetManifestStr) | ||
| By("Applying Taget CNI Manifest") | ||
| common.ApplyCNIManifest(targetManifest) | ||
| common.ValidateTraffic(f, serverDeploymentBuilder, 99, "tcp") | ||
| }) | ||
| }) | ||
|
|
||
| Context("when testing UDP traffic between client and server pods", func() { | ||
| BeforeEach(func() { | ||
| enableIPv4PrefixDelegation = "true" | ||
| }) | ||
|
|
||
| //TODO : Add pod IP validation if IP belongs to prefix or SIP | ||
| //TODO : remove hardcoding from client/server count | ||
| It("should have 99+% success rate", func() { | ||
| By("test with initial cni manifest file") | ||
| common.ValidateTraffic(f, serverDeploymentBuilder, 99, "udp") | ||
| targetManifestStr := fmt.Sprintf("Testing with Target CNI Manifest: %s", targetManifest) | ||
| By(targetManifestStr) | ||
| By("Applying Taget CNI Manifest") | ||
| common.ApplyCNIManifest(targetManifest) | ||
| common.ValidateTraffic(f, serverDeploymentBuilder, 99, "udp") | ||
| }) | ||
| }) | ||
| }) |
116 changes: 116 additions & 0 deletions
116
test/integration/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| package cni_upgrade_downgrade | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
|
|
||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework" | ||
| k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
| "github.com/pkg/errors" | ||
|
|
||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
| v1 "k8s.io/api/core/v1" | ||
| ) | ||
|
|
||
| const ( | ||
| InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type" | ||
| DefaultManifestfile = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/master/config/master/aws-k8s-cni.yaml" | ||
| ) | ||
|
|
||
| var f *framework.Framework | ||
| var maxIPPerInterface int | ||
| var primaryNode v1.Node | ||
| var secondaryNode v1.Node | ||
| var instanceSecurityGroupID string | ||
| var vpcCIDRs []string | ||
| var initialManifest string | ||
| var targetManifest string | ||
|
|
||
| func TestCNIPodNetworking(t *testing.T) { | ||
| RegisterFailHandler(Fail) | ||
| RunSpecs(t, "CNI Pod Networking Suite") | ||
| } | ||
|
|
||
| var _ = BeforeSuite(func() { | ||
| f = framework.New(framework.GlobalOptions) | ||
|
|
||
| By("creating test namespace") | ||
| f.K8sResourceManagers.NamespaceManager(). | ||
| CreateNamespace(utils.DefaultTestNamespace) | ||
|
|
||
| By(fmt.Sprintf("getting the node with the node label key %s and value %s", | ||
| f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)) | ||
| nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| By("verifying more than 1 nodes are present for the test") | ||
| Expect(len(nodes.Items)).Should(BeNumerically(">", 1)) | ||
|
|
||
| // Set the primary and secondary node for testing | ||
| primaryNode = nodes.Items[0] | ||
| secondaryNode = nodes.Items[1] | ||
|
|
||
| // Get the node security group | ||
| instanceID := k8sUtils.GetInstanceIDFromNode(primaryNode) | ||
| primaryInstance, err := f.CloudServices.EC2().DescribeInstance(instanceID) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| // This won't work if the first SG is only associated with the primary instance. | ||
| // Need a robust substring in the SGP name to identify node SGP | ||
| instanceSecurityGroupID = *primaryInstance.NetworkInterfaces[0].Groups[0].GroupId | ||
|
|
||
| By("getting the instance type from node label " + InstanceTypeNodeLabelKey) | ||
| instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey] | ||
|
|
||
| By("getting the network interface details from ec2") | ||
| instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| // Subtract 2 for coredns pods if present, and both could be on same ENI | ||
| maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface) - 2 | ||
|
|
||
| By("describing the VPC to get the VPC CIDRs") | ||
| describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| for _, cidrBlockAssociationSet := range describeVPCOutput.Vpcs[0].CidrBlockAssociationSet { | ||
| vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock) | ||
| } | ||
|
|
||
| initialManifest = f.Options.InitialManifest | ||
| targetManifest = f.Options.TargetManifest | ||
| if len(targetManifest) == 0 { | ||
| err = errors.Errorf("Target Manifest file must be specified") | ||
| } | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| if len(initialManifest) == 0 { | ||
| initialManifest = DefaultManifestfile | ||
| } | ||
|
|
||
| initialManifestStr := fmt.Sprintf("using initial cni manifest: %s", initialManifest) | ||
| targetManifestStr := fmt.Sprintf("using target cni manifest: %s", targetManifest) | ||
|
|
||
| By(initialManifestStr) | ||
| By(targetManifestStr) | ||
|
|
||
| // Set the WARM_ENI_TARGET to 0 to prevent all pods being scheduled on secondary ENI | ||
| k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node", "kube-system", | ||
| "aws-node", map[string]string{"WARM_IP_TARGET": "3", "WARM_ENI_TARGET": "0"}) | ||
| }) | ||
|
|
||
| var _ = AfterSuite(func() { | ||
| By("deleting test namespace") | ||
| f.K8sResourceManagers.NamespaceManager(). | ||
| DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) | ||
|
|
||
| k8sUtils.UpdateEnvVarOnDaemonSetAndWaitUntilReady(f, "aws-node", "kube-system", | ||
| "aws-node", | ||
| nil, | ||
| map[string]struct{}{ | ||
| "WARM_IP_TARGET": {}, | ||
| "WARM_ENI_TARGET": {}, | ||
| }) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.