forked from aws/amazon-vpc-cni-k8s
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Addon upgrade/downgrade test similar to aws#1795 Added tests for addon upgrade/downgrade Changed DEFAULT version Added addon status checks Fetch latest addon version for given K8s Cluster Update kops cluster config used in weekly tests (aws#1862) * Change to kops cluster creation scripts * Add logging for retry attempt * Switch kops cluster to use docker container runtime Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Renamed package name for adddon tests removed unnecessary changes Fixed replica count for MTU and Veth test in host networking Updated ENI/IP limits file for newly added instances (aws#1864) * Added new instances * Updated test readme * needed rebase * formatting
- Loading branch information
Showing
9 changed files
with
520 additions
and
195 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package utils | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
| "github.com/pkg/errors" | ||
| "k8s.io/apimachinery/pkg/util/wait" | ||
| ) | ||
|
|
||
| func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error { | ||
| ctx := context.Background() | ||
| return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { | ||
| _, err := eks.DescribeAddon(&services.AddonInput{ | ||
| AddonName: addonName, | ||
| ClusterName: clusterName, | ||
| }) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| return false, nil | ||
| }, ctx.Done()) | ||
| } | ||
|
|
||
| func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error { | ||
| ctx := context.Background() | ||
| return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { | ||
| describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{ | ||
| AddonName: addonName, | ||
| ClusterName: clusterName, | ||
| }) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| status := *describeAddonOutput.Addon.Status | ||
| if status == "CREATE_FAILED" || status == "DEGRADED" { | ||
| return false, errors.Errorf("Create Addon Failed, addon status: %s", status) | ||
| } | ||
| if status == "ACTIVE" { | ||
| return true, nil | ||
| } | ||
| return false, nil | ||
| }, ctx.Done()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| package addon_tests | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
|
|
||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
| v1 "k8s.io/api/core/v1" | ||
|
|
||
| . "github.com/onsi/ginkgo" | ||
| . "github.com/onsi/gomega" | ||
| ) | ||
|
|
||
| const InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type" | ||
|
|
||
| var f *framework.Framework | ||
| var maxIPPerInterface int | ||
| var primaryNode v1.Node | ||
| var secondaryNode v1.Node | ||
| var vpcCIDRs []string | ||
| var clusterVersion string | ||
| var latestAddonVersion string | ||
|
|
||
| func TestCNIPodNetworking(t *testing.T) { | ||
| RegisterFailHandler(Fail) | ||
| RunSpecs(t, "CNI Pod Networking Suite") | ||
| } | ||
|
|
||
| var _ = BeforeSuite(func() { | ||
| f = framework.New(framework.GlobalOptions) | ||
|
|
||
| By("creating test namespace") | ||
| f.K8sResourceManagers.NamespaceManager(). | ||
| CreateNamespace(utils.DefaultTestNamespace) | ||
|
|
||
| By(fmt.Sprintf("getting the node with the node label key %s and value %s", | ||
| f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)) | ||
| nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| By("verifying that atleast 1 nodes is present for the test") | ||
| Expect(len(nodes.Items)).Should(BeNumerically(">", 0)) | ||
|
|
||
| // Set the primary and secondary node for testing | ||
| primaryNode = nodes.Items[0] | ||
| secondaryNode = nodes.Items[1] | ||
|
|
||
| By("getting the instance type from node label " + InstanceTypeNodeLabelKey) | ||
| instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey] | ||
|
|
||
| By("getting the network interface details from ec2") | ||
| instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface) | ||
|
|
||
| By("describing the VPC to get the VPC CIDRs") | ||
| describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| for _, cidrBlockAssociationSet := range describeVPCOutput.Vpcs[0].CidrBlockAssociationSet { | ||
| vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock) | ||
| } | ||
|
|
||
| By("getting current cluster version") | ||
| clusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| clusterVersion = *clusterOutput.Cluster.Version | ||
|
|
||
| By("getting latest vpc-cni addon version") | ||
| latestAddonVersion, err = f.CloudServices.EKS().GetLatestVersion(&services.AddonInput{ | ||
| AddonName: "vpc-cni", | ||
| K8sVersion: clusterVersion, | ||
| }) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| }) | ||
|
|
||
| var _ = AfterSuite(func() { | ||
| By("deleting test namespace") | ||
| f.K8sResourceManagers.NamespaceManager(). | ||
| DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| package addon_tests | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" | ||
| k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
| "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" | ||
| . "github.com/onsi/ginkgo" | ||
| . "github.com/onsi/gomega" | ||
| v1 "k8s.io/api/apps/v1" | ||
| ) | ||
|
|
||
| var ( | ||
| err error | ||
| podLabelKey = "app" | ||
| podLabelVal = "host-networking-test" | ||
| deployment *v1.Deployment | ||
| ) | ||
|
|
||
| const ( | ||
| // Using v1.9.x as the default since v1.7.5 has following issue:https://github.com/aws/amazon-vpc-cni-k8s/pull/1341 | ||
| DEFAULT_VERSION = "v1.9.1-eksbuild.1" | ||
| ) | ||
|
|
||
| var _ = Describe("cni addon upgrade/downgrade test", func() { | ||
| Context("test host networking", func() { | ||
| It("create test deployment with initial addon and verify deletion with target addon", func() { | ||
| targetAddonVersion := f.Options.TargetAddon | ||
| initialAddonVersion := f.Options.InitialAddon | ||
| if len(f.Options.TargetAddon) == 0 { | ||
| targetAddonVersion = latestAddonVersion | ||
| } | ||
| By(fmt.Sprintf("using target addon version as %s", targetAddonVersion)) | ||
|
|
||
| if len(f.Options.InitialAddon) == 0 { | ||
| initialAddonVersion = DEFAULT_VERSION | ||
| } | ||
| By(fmt.Sprintf("using initial addon version as %s", initialAddonVersion)) | ||
|
|
||
| ApplyAddOn(initialAddonVersion) | ||
|
|
||
| interfaceTypeToPodList := getTestPodList() | ||
| By("generating the pod networking validation input to be passed to tester") | ||
| input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| By("validating host networking is setup correctly with initial addon") | ||
| common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) | ||
|
|
||
| By("upgrading to target addon version") | ||
| ApplyAddOn(targetAddonVersion) | ||
|
|
||
| deleteTestDeployment() | ||
|
|
||
| By("validating host networking is teared down correctly with target addon") | ||
| common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) | ||
| }) | ||
|
|
||
| It("create test deployment with target addon and verify deletion with initial addon", func() { | ||
| targetAddonVersion := f.Options.TargetAddon | ||
| initialAddonVersion := f.Options.InitialAddon | ||
| if len(f.Options.TargetAddon) == 0 { | ||
| targetAddonVersion = latestAddonVersion | ||
| } | ||
| By(fmt.Sprintf("using target addon version as %s", targetAddonVersion)) | ||
|
|
||
| if len(f.Options.InitialAddon) == 0 { | ||
| initialAddonVersion = DEFAULT_VERSION | ||
| } | ||
| By(fmt.Sprintf("using initial addon version as %s", initialAddonVersion)) | ||
|
|
||
| ApplyAddOn(targetAddonVersion) | ||
|
|
||
| interfaceTypeToPodList := getTestPodList() | ||
| By("generating the pod networking validation input to be passed to tester") | ||
| input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| By("validating host networking is setup correctly with target addon") | ||
| common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) | ||
|
|
||
| By("downgrading to initial addon version") | ||
| ApplyAddOn(initialAddonVersion) | ||
|
|
||
| deleteTestDeployment() | ||
|
|
||
| By("validating host networking is teared down correctly with initial addon") | ||
| common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) | ||
| }) | ||
| }) | ||
| }) | ||
|
|
||
| func deleteTestDeployment() { | ||
| By("deleting the deployment to test teardown") | ||
| err = f.K8sResourceManagers.DeploymentManager(). | ||
| DeleteAndWaitTillDeploymentIsDeleted(deployment) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| By("waiting to allow CNI to tear down networking for terminated pods") | ||
| time.Sleep(time.Second * 60) | ||
| } | ||
|
|
||
| func getTestPodList() common.InterfaceTypeToPodList { | ||
| deployment = manifest.NewBusyBoxDeploymentBuilder(). | ||
| Replicas(maxIPPerInterface*2). | ||
| PodLabel(podLabelKey, podLabelVal). | ||
| NodeName(primaryNode.Name). | ||
| Build() | ||
|
|
||
| By("creating a deployment to launch pod using primary and secondary ENI IP") | ||
| _, err := f.K8sResourceManagers.DeploymentManager(). | ||
| CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| By("getting the list of pods using IP from primary and secondary ENI") | ||
| interfaceTypeToPodList := | ||
| common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) | ||
|
|
||
| return interfaceTypeToPodList | ||
| } | ||
|
|
||
| func ApplyAddOn(version string) { | ||
| By("delete existing cni addon if any") | ||
| _, err := f.CloudServices.EKS().DescribeAddon(&services.AddonInput{ | ||
| AddonName: "vpc-cni", | ||
| ClusterName: f.Options.ClusterName, | ||
| }) | ||
| if err == nil { | ||
| _, err := f.CloudServices.EKS().DeleteAddon(&services.AddonInput{ | ||
| AddonName: "vpc-cni", | ||
| ClusterName: f.Options.ClusterName, | ||
| }) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| By("wait till the old addon is deleted") | ||
| err = k8sUtils.WaitTillAddonIsDeleted(f.CloudServices.EKS(), "vpc-cni", f.Options.ClusterName) | ||
| Expect(err).To(HaveOccurred()) | ||
| } | ||
| By(fmt.Sprintf("install the new addon with version: %s", version)) | ||
| _, err = f.CloudServices.EKS().CreateAddon(&services.AddonInput{ | ||
| AddonName: "vpc-cni", | ||
| ClusterName: f.Options.ClusterName, | ||
| AddonVersion: version, | ||
| }) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| By("wait till the addon is active") | ||
| err = k8sUtils.WaitTillAddonIsActive(f.CloudServices.EKS(), "vpc-cni", f.Options.ClusterName) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| } |
Oops, something went wrong.