diff --git a/test/framework/options.go b/test/framework/options.go index 4c7cf34065..e93fd30ce4 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -34,6 +34,8 @@ type Options struct { NgNameLabelKey string NgNameLabelVal string EKSEndpoint string + InitialAddon string + TargetAddon string } func (options *Options) BindFlags() { @@ -44,6 +46,8 @@ func (options *Options) BindFlags() { flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name") flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name") flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint") + flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied") + flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied") } func (options *Options) Validate() error { diff --git a/test/framework/resources/aws/services/eks.go b/test/framework/resources/aws/services/eks.go index 838870d9b4..1b1b35bf64 100644 --- a/test/framework/resources/aws/services/eks.go +++ b/test/framework/resources/aws/services/eks.go @@ -22,14 +22,27 @@ import ( type EKS interface { DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) - CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) - DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) + CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) + DescribeAddonVersions(AddonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) + DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) + DeleteAddon(AddOnInput *AddonInput) (*eks.DeleteAddonOutput, error) + GetLatestVersion(addonInput *AddonInput) (string, error) } type defaultEKS struct { eksiface.EKSAPI } +// Internal Addon Input struct +// subset of eks.AddonInput +// used by ginkgo tests +type AddonInput struct { + AddonName string + ClusterName string + AddonVersion string + K8sVersion string +} + func NewEKS(session *session.Session, endpoint string) EKS { return &defaultEKS{ EKSAPI: eks.New(session, &aws.Config{ @@ -39,22 +52,42 @@ func NewEKS(session *session.Session, endpoint string) EKS { } } -func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) { +func (d defaultEKS) CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) { createAddonInput := &eks.CreateAddonInput{ - AddonName: aws.String(addon), - ClusterName: aws.String(clusterName), + AddonName: aws.String(addonInput.AddonName), + ClusterName: aws.String(addonInput.ClusterName), + } + if addonInput.AddonVersion != "" { + createAddonInput.SetAddonVersion(addonInput.AddonVersion) + createAddonInput.SetResolveConflicts("OVERWRITE") } return d.EKSAPI.CreateAddon(createAddonInput) } -func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) { +func (d defaultEKS) DeleteAddon(addonInput *AddonInput) (*eks.DeleteAddonOutput, error) { deleteAddonInput := &eks.DeleteAddonInput{ - AddonName: aws.String(addon), - ClusterName: aws.String(clusterName), + AddonName: aws.String(addonInput.AddonName), + ClusterName: aws.String(addonInput.ClusterName), } return d.EKSAPI.DeleteAddon(deleteAddonInput) } +func (d defaultEKS) DescribeAddonVersions(addonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) { + describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{ + AddonName: aws.String(addonInput.AddonName), + KubernetesVersion: aws.String(addonInput.K8sVersion), + } + return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput) +} + +func (d defaultEKS) DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) { + describeAddonInput := &eks.DescribeAddonInput{ + AddonName: aws.String(addonInput.AddonName), + ClusterName: aws.String(addonInput.ClusterName), + } + return d.EKSAPI.DescribeAddon(describeAddonInput) +} + func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) { describeClusterInput := &eks.DescribeClusterInput{ Name: aws.String(clusterName), @@ -62,3 +95,11 @@ func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOut return d.EKSAPI.DescribeCluster(describeClusterInput) } + +func (d defaultEKS) GetLatestVersion(addonInput *AddonInput) (string, error) { + addonOutput, err := d.DescribeAddonVersions(addonInput) + if err != nil { + return "", err + } + return *addonOutput.Addons[0].AddonVersions[0].AddonVersion, nil +} diff --git a/test/framework/resources/k8s/utils/addon.go b/test/framework/resources/k8s/utils/addon.go new file mode 100644 index 0000000000..6bd12d590b --- /dev/null +++ b/test/framework/resources/k8s/utils/addon.go @@ -0,0 +1,46 @@ +package utils + +import ( + "context" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/wait" +) + +func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error { + ctx := context.Background() + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + _, err := eks.DescribeAddon(&services.AddonInput{ + AddonName: addonName, + ClusterName: clusterName, + }) + if err != nil { + return false, err + } + return false, nil + }, ctx.Done()) +} + +func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error { + ctx := context.Background() + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{ + AddonName: addonName, + ClusterName: clusterName, + }) + if err != nil { + return false, err + } + + status := *describeAddonOutput.Addon.Status + if status == "CREATE_FAILED" || status == "DEGRADED" { + return false, errors.Errorf("Create Addon Failed, addon status: %s", status) + } + if status == "ACTIVE" { + return true, nil + } + return false, nil + }, ctx.Done()) +} diff --git a/test/integration-new/addon-tests/cni_addon_suite_test.go b/test/integration-new/addon-tests/cni_addon_suite_test.go new file mode 100644 index 0000000000..04f41d7736 --- /dev/null +++ b/test/integration-new/addon-tests/cni_addon_suite_test.go @@ -0,0 +1,84 @@ +package addon_tests + +import ( + "fmt" + "testing" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + v1 "k8s.io/api/core/v1" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type" + +var f *framework.Framework +var maxIPPerInterface int +var primaryNode v1.Node +var secondaryNode v1.Node +var vpcCIDRs []string +var clusterVersion string +var latestAddonVersion string + +func TestCNIPodNetworking(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CNI Pod Networking Suite") +} + +var _ = BeforeSuite(func() { + f = framework.New(framework.GlobalOptions) + + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + + By(fmt.Sprintf("getting the node with the node label key %s and value %s", + f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)) + nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + Expect(err).ToNot(HaveOccurred()) + + By("verifying that atleast 1 nodes is present for the test") + Expect(len(nodes.Items)).Should(BeNumerically(">", 0)) + + // Set the primary and secondary node for testing + primaryNode = nodes.Items[0] + secondaryNode = nodes.Items[1] + + By("getting the instance type from node label " + InstanceTypeNodeLabelKey) + instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey] + + By("getting the network interface details from ec2") + instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType) + Expect(err).ToNot(HaveOccurred()) + + maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface) + + By("describing the VPC to get the VPC CIDRs") + describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + + for _, cidrBlockAssociationSet := range describeVPCOutput.Vpcs[0].CidrBlockAssociationSet { + vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock) + } + + By("getting current cluster version") + clusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) + Expect(err).NotTo(HaveOccurred()) + clusterVersion = *clusterOutput.Cluster.Version + + By("getting latest vpc-cni addon version") + latestAddonVersion, err = f.CloudServices.EKS().GetLatestVersion(&services.AddonInput{ + AddonName: "vpc-cni", + K8sVersion: clusterVersion, + }) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) +}) diff --git a/test/integration-new/addon-tests/cni_addon_test.go b/test/integration-new/addon-tests/cni_addon_test.go new file mode 100644 index 0000000000..9a86804508 --- /dev/null +++ b/test/integration-new/addon-tests/cni_addon_test.go @@ -0,0 +1,154 @@ +package addon_tests + +import ( + "fmt" + "time" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" +) + +var ( + err error + podLabelKey = "app" + podLabelVal = "host-networking-test" + deployment *v1.Deployment +) + +const ( + // Using v1.9.x as the default since v1.7.5 has following issue:https://github.com/aws/amazon-vpc-cni-k8s/pull/1341 + DEFAULT_VERSION = "v1.9.1-eksbuild.1" +) + +var _ = Describe("cni addon upgrade/downgrade test", func() { + Context("test host networking", func() { + It("create test deployment with initial addon and verify deletion with target addon", func() { + targetAddonVersion := f.Options.TargetAddon + initialAddonVersion := f.Options.InitialAddon + if len(f.Options.TargetAddon) == 0 { + targetAddonVersion = latestAddonVersion + } + By(fmt.Sprintf("using target addon version as %s", targetAddonVersion)) + + if len(f.Options.InitialAddon) == 0 { + initialAddonVersion = DEFAULT_VERSION + } + By(fmt.Sprintf("using initial addon version as %s", initialAddonVersion)) + + ApplyAddOn(initialAddonVersion) + + interfaceTypeToPodList := getTestPodList() + By("generating the pod networking validation input to be passed to tester") + input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() + Expect(err).NotTo(HaveOccurred()) + + By("validating host networking is setup correctly with initial addon") + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) + + By("upgrading to target addon version") + ApplyAddOn(targetAddonVersion) + + deleteTestDeployment() + + By("validating host networking is teared down correctly with target addon") + common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) + }) + + It("create test deployment with target addon and verify deletion with initial addon", func() { + targetAddonVersion := f.Options.TargetAddon + initialAddonVersion := f.Options.InitialAddon + if len(f.Options.TargetAddon) == 0 { + targetAddonVersion = latestAddonVersion + } + By(fmt.Sprintf("using target addon version as %s", targetAddonVersion)) + + if len(f.Options.InitialAddon) == 0 { + initialAddonVersion = DEFAULT_VERSION + } + By(fmt.Sprintf("using initial addon version as %s", initialAddonVersion)) + + ApplyAddOn(targetAddonVersion) + + interfaceTypeToPodList := getTestPodList() + By("generating the pod networking validation input to be passed to tester") + input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() + Expect(err).NotTo(HaveOccurred()) + + By("validating host networking is setup correctly with target addon") + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) + + By("downgrading to initial addon version") + ApplyAddOn(initialAddonVersion) + + deleteTestDeployment() + + By("validating host networking is teared down correctly with initial addon") + common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) + }) + }) +}) + +func deleteTestDeployment() { + By("deleting the deployment to test teardown") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("waiting to allow CNI to tear down networking for terminated pods") + time.Sleep(time.Second * 60) +} + +func getTestPodList() common.InterfaceTypeToPodList { + deployment = manifest.NewBusyBoxDeploymentBuilder(). + Replicas(maxIPPerInterface*2). + PodLabel(podLabelKey, podLabelVal). + NodeName(primaryNode.Name). + Build() + + By("creating a deployment to launch pod using primary and secondary ENI IP") + _, err := f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("getting the list of pods using IP from primary and secondary ENI") + interfaceTypeToPodList := + common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) + + return interfaceTypeToPodList +} + +func ApplyAddOn(version string) { + By("delete existing cni addon if any") + _, err := f.CloudServices.EKS().DescribeAddon(&services.AddonInput{ + AddonName: "vpc-cni", + ClusterName: f.Options.ClusterName, + }) + if err == nil { + _, err := f.CloudServices.EKS().DeleteAddon(&services.AddonInput{ + AddonName: "vpc-cni", + ClusterName: f.Options.ClusterName, + }) + Expect(err).NotTo(HaveOccurred()) + + By("wait till the old addon is deleted") + err = k8sUtils.WaitTillAddonIsDeleted(f.CloudServices.EKS(), "vpc-cni", f.Options.ClusterName) + Expect(err).To(HaveOccurred()) + } + By(fmt.Sprintf("install the new addon with version: %s", version)) + _, err = f.CloudServices.EKS().CreateAddon(&services.AddonInput{ + AddonName: "vpc-cni", + ClusterName: f.Options.ClusterName, + AddonVersion: version, + }) + Expect(err).NotTo(HaveOccurred()) + + By("wait till the addon is active") + err = k8sUtils.WaitTillAddonIsActive(f.CloudServices.EKS(), "vpc-cni", f.Options.ClusterName) + Expect(err).NotTo(HaveOccurred()) +} diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index 03a3abe687..ea4d99a074 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -14,32 +14,21 @@ package cni import ( - "fmt" "strconv" "time" - "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" v1 "k8s.io/api/core/v1" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) // TODO: Instead of passing the list of pods to the test helper, have the test helper get the pod on node - -type TestType int - -const ( - NetworkingTearDownSucceeds TestType = iota - NetworkingTearDownFails - NetworkingSetupSucceeds - NetworkingSetupFails -) - const ( AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU" AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX" @@ -76,8 +65,7 @@ var _ = Describe("test host networking", func() { Expect(err).ToNot(HaveOccurred()) By("getting the list of pods using IP from primary and secondary ENI") - interfaceTypeToPodList := - GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal) + interfaceTypeToPodList := common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) // Primary ENI and Secondary ENI IPs are handled differently when setting up // the host networking rule hence this check @@ -87,11 +75,11 @@ var _ = Describe("test host networking", func() { Should(BeNumerically(">", 0)) By("generating the pod networking validation input to be passed to tester") - input, err := GetPodNetworkingValidationInput(interfaceTypeToPodList).Serialize() + input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) By("validating host networking setup is setup correctly") - ValidateHostNetworking(NetworkingSetupSucceeds, input) + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) By("deleting the deployment to test teardown") err = f.K8sResourceManagers.DeploymentManager(). @@ -102,12 +90,12 @@ var _ = Describe("test host networking", func() { time.Sleep(time.Second * 60) By("validating host networking is teared down correctly") - ValidateHostNetworking(NetworkingTearDownSucceeds, input) + common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) }) It("Validate Host Networking setup after changing MTU and Veth Prefix", func() { deployment := manifest.NewBusyBoxDeploymentBuilder(). - Replicas(6). + Replicas(maxIPPerInterface*2). PodLabel(podLabelKey, podLabelVal). NodeName(primaryNode.Name). Build() @@ -125,10 +113,10 @@ var _ = Describe("test host networking", func() { By("getting the list of pods using IP from primary and secondary ENI") interfaceTypeToPodList := - GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal) + common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) By("generating the pod networking validation input to be passed to tester") - podNetworkingValidationInput := GetPodNetworkingValidationInput(interfaceTypeToPodList) + podNetworkingValidationInput := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs) podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX podNetworkingValidationInput.ValidateMTU = true podNetworkingValidationInput.MTU = NEW_MTU_VAL @@ -136,7 +124,7 @@ var _ = Describe("test host networking", func() { Expect(err).NotTo(HaveOccurred()) By("validating host networking setup is setup correctly with MTU check as well") - ValidateHostNetworking(NetworkingSetupSucceeds, input) + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) By("deleting the deployment to test teardown") err = f.K8sResourceManagers.DeploymentManager(). @@ -147,7 +135,7 @@ var _ = Describe("test host networking", func() { time.Sleep(time.Second * 60) By("validating host networking is teared down correctly") - ValidateHostNetworking(NetworkingTearDownSucceeds, input) + common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) }) }) @@ -165,39 +153,39 @@ var _ = Describe("test host networking", func() { CreatAndWaitTillRunning(parkingPod) Expect(err).ToNot(HaveOccurred()) - validInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + validInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*parkingPod}, - }).Serialize() + }, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) By("first validating the tester work on valid input") - ValidateHostNetworking(NetworkingSetupSucceeds, validInput) + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, validInput, primaryNode.Name, f) By("validating tester fails when invalid IP is passed") invalidPod := parkingPod.DeepCopy() invalidPod.Status.PodIP = "1.1.1.1" - invalidInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }).Serialize() + }, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) - ValidateHostNetworking(NetworkingSetupFails, invalidInput) + common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f) By("validating the tester fails when invalid namespace is passed") invalidPod = parkingPod.DeepCopy() // veth pair name is generated using namespace+name so the test should fail invalidPod.Namespace = "different" - invalidInput, err = GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err = common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }).Serialize() + }, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) - ValidateHostNetworking(NetworkingSetupFails, invalidInput) + common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f) By("validating the tester fails when tear down check is run on running pod") - ValidateHostNetworking(NetworkingTearDownFails, validInput) + common.ValidateHostNetworking(common.NetworkingTearDownFails, validInput, primaryNode.Name, f) By("deleting the parking pod") err = f.K8sResourceManagers.PodManager(). @@ -206,85 +194,3 @@ var _ = Describe("test host networking", func() { }) }) }) - -// Validate host networking for the list of pods supplied -func ValidateHostNetworking(testType TestType, podValidationInputString string) { - testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s", - podValidationInputString)} - - var shouldTestPodError bool - if NetworkingSetupSucceeds == testType { - testerArgs = append(testerArgs, "-test-setup=true") - } else if NetworkingSetupFails == testType { - testerArgs = append(testerArgs, "-test-setup=true") - shouldTestPodError = true - } else if NetworkingTearDownSucceeds == testType { - testerArgs = append(testerArgs, "-test-cleanup=true") - } else if NetworkingTearDownFails == testType { - testerArgs = append(testerArgs, "-test-cleanup=true") - shouldTestPodError = true - } - - testContainer := manifest.NewTestHelperContainer(). - Command([]string{"./networking"}). - Args(testerArgs). - Build() - - testPod := manifest.NewDefaultPodBuilder(). - Container(testContainer). - NodeName(primaryNode.Name). - HostNetwork(true). - Build() - - By("creating pod to test host networking setup") - testPod, err := f.K8sResourceManagers.PodManager(). - CreateAndWaitTillPodCompleted(testPod) - logs, errLogs := f.K8sResourceManagers.PodManager(). - PodLogs(testPod.Namespace, testPod.Name) - Expect(errLogs).ToNot(HaveOccurred()) - - fmt.Fprintln(GinkgoWriter, logs) - - if shouldTestPodError { - Expect(err).To(HaveOccurred()) - } else { - Expect(err).ToNot(HaveOccurred()) - } - - By("deleting the host networking setup pod") - err = f.K8sResourceManagers.PodManager(). - DeleteAndWaitTillPodDeleted(testPod) - Expect(err).ToNot(HaveOccurred()) -} - -// GetPodNetworkingValidationInput returns input string containing the list of pods for which -// the host networking has to be tested -func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) input.PodNetworkingValidationInput { - - ip := input.PodNetworkingValidationInput{ - VPCCidrRange: vpcCIDRs, - VethPrefix: "eni", - PodList: []input.Pod{}, - ValidateMTU: false, - } - - for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI { - ip.PodList = append(ip.PodList, input.Pod{ - PodName: primaryENIPod.Name, - PodNamespace: primaryENIPod.Namespace, - PodIPv4Address: primaryENIPod.Status.PodIP, - IsIPFromSecondaryENI: false, - }) - } - - for _, secondaryENIPod := range interfaceTypeToPodList.PodsOnSecondaryENI { - ip.PodList = append(ip.PodList, input.Pod{ - PodName: secondaryENIPod.Name, - PodNamespace: secondaryENIPod.Namespace, - PodIPv4Address: secondaryENIPod.Status.PodIP, - IsIPFromSecondaryENI: true, - }) - } - - return ip -} diff --git a/test/integration-new/cni/pod_traffic_test.go b/test/integration-new/cni/pod_traffic_test.go index 7014ddac61..f832f5b532 100644 --- a/test/integration-new/cni/pod_traffic_test.go +++ b/test/integration-new/cni/pod_traffic_test.go @@ -18,9 +18,9 @@ import ( "strconv" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" - k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo" @@ -64,9 +64,9 @@ var _ = Describe("test pod networking", func() { secondaryNodeDeployment *v1.Deployment // Map of Pods placed on primary/secondary ENI IP on primary node - interfaceToPodListOnPrimaryNode InterfaceTypeToPodList + interfaceToPodListOnPrimaryNode common.InterfaceTypeToPodList // Map of Pods placed on primary/secondary ENI IP on secondary node - interfaceToPodListOnSecondaryNode InterfaceTypeToPodList + interfaceToPodListOnSecondaryNode common.InterfaceTypeToPodList ) JustBeforeEach(func() { @@ -102,7 +102,7 @@ var _ = Describe("test pod networking", func() { Expect(err).ToNot(HaveOccurred()) interfaceToPodListOnPrimaryNode = - GetPodsOnPrimaryAndSecondaryInterface(primaryNode, "node", "primary") + common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, "node", "primary", f) // At least two Pods should be placed on the Primary and Secondary Interface // on the Primary and Secondary Node in order to test all possible scenarios @@ -127,7 +127,7 @@ var _ = Describe("test pod networking", func() { Expect(err).ToNot(HaveOccurred()) interfaceToPodListOnSecondaryNode = - GetPodsOnPrimaryAndSecondaryInterface(secondaryNode, "node", "secondary") + common.GetPodsOnPrimaryAndSecondaryInterface(secondaryNode, "node", "secondary", f) // Same reason as mentioned above Expect(len(interfaceToPodListOnSecondaryNode.PodsOnPrimaryENI)). @@ -282,8 +282,8 @@ func VerifyConnectivityFailsForNegativeCase(senderPod coreV1.Pod, receiverPod co // CheckConnectivityForMultiplePodPlacement checks connectivity for various scenarios, an example // connection from Pod on Node 1 having IP from Primary Network Interface to Pod on Node 2 having // IP from Secondary Network Interface -func CheckConnectivityForMultiplePodPlacement(interfaceToPodListOnPrimaryNode InterfaceTypeToPodList, - interfaceToPodListOnSecondaryNode InterfaceTypeToPodList, port int, +func CheckConnectivityForMultiplePodPlacement(interfaceToPodListOnPrimaryNode common.InterfaceTypeToPodList, + interfaceToPodListOnSecondaryNode common.InterfaceTypeToPodList, port int, testerExpectedStdOut string, testerExpectedStdErr string, getTestCommandFunc func(receiverPod coreV1.Pod, port int) []string) { @@ -343,57 +343,3 @@ func testConnectivity(senderPod coreV1.Pod, receiverPod coreV1.Pod, expectedStdo Expect(stdErr).To(ContainSubstring(expectedStderr)) Expect(stdOut).To(ContainSubstring(expectedStdout)) } - -type InterfaceTypeToPodList struct { - PodsOnPrimaryENI []coreV1.Pod - PodsOnSecondaryENI []coreV1.Pod -} - -// GetPodsOnPrimaryAndSecondaryInterface returns the list of Pods on Primary Networking -// Interface and Secondary Network Interface on a given Node -func GetPodsOnPrimaryAndSecondaryInterface(node coreV1.Node, - podLabelKey string, podLabelVal string) InterfaceTypeToPodList { - podList, err := f.K8sResourceManagers. - PodManager(). - GetPodsWithLabelSelector(podLabelKey, podLabelVal) - Expect(err).ToNot(HaveOccurred()) - - instance, err := f.CloudServices.EC2(). - DescribeInstance(k8sUtils.GetInstanceIDFromNode(node)) - Expect(err).ToNot(HaveOccurred()) - - interfaceToPodList := InterfaceTypeToPodList{ - PodsOnPrimaryENI: []coreV1.Pod{}, - PodsOnSecondaryENI: []coreV1.Pod{}, - } - - ipToPod := map[string]coreV1.Pod{} - for _, pod := range podList.Items { - ipToPod[pod.Status.PodIP] = pod - } - - for _, nwInterface := range instance.NetworkInterfaces { - isPrimary := IsPrimaryENI(nwInterface, instance.PrivateIpAddress) - for _, ip := range nwInterface.PrivateIpAddresses { - if pod, found := ipToPod[*ip.PrivateIpAddress]; found { - if isPrimary { - interfaceToPodList.PodsOnPrimaryENI = - append(interfaceToPodList.PodsOnPrimaryENI, pod) - } else { - interfaceToPodList.PodsOnSecondaryENI = - append(interfaceToPodList.PodsOnSecondaryENI, pod) - } - } - } - } - return interfaceToPodList -} - -func IsPrimaryENI(nwInterface *ec2.InstanceNetworkInterface, instanceIPAddr *string) bool { - for _, privateIPAddress := range nwInterface.PrivateIpAddresses { - if *privateIPAddress.PrivateIpAddress == *instanceIPAddr { - return true - } - } - return false -} diff --git a/test/integration-new/common/util.go b/test/integration-new/common/util.go new file mode 100644 index 0000000000..ea777aebb3 --- /dev/null +++ b/test/integration-new/common/util.go @@ -0,0 +1,156 @@ +package common + +import ( + "fmt" + + "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + coreV1 "k8s.io/api/core/v1" +) + +type TestType int + +const ( + NetworkingTearDownSucceeds TestType = iota + NetworkingTearDownFails + NetworkingSetupSucceeds + NetworkingSetupFails +) + +type InterfaceTypeToPodList struct { + PodsOnPrimaryENI []coreV1.Pod + PodsOnSecondaryENI []coreV1.Pod +} + +func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList, vpcCIDRs []string) input.PodNetworkingValidationInput { + + ip := input.PodNetworkingValidationInput{ + VPCCidrRange: vpcCIDRs, + VethPrefix: "eni", + PodList: []input.Pod{}, + ValidateMTU: false, + } + + for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI { + ip.PodList = append(ip.PodList, input.Pod{ + PodName: primaryENIPod.Name, + PodNamespace: primaryENIPod.Namespace, + PodIPv4Address: primaryENIPod.Status.PodIP, + IsIPFromSecondaryENI: false, + }) + } + + for _, secondaryENIPod := range interfaceTypeToPodList.PodsOnSecondaryENI { + ip.PodList = append(ip.PodList, input.Pod{ + PodName: secondaryENIPod.Name, + PodNamespace: secondaryENIPod.Namespace, + PodIPv4Address: secondaryENIPod.Status.PodIP, + IsIPFromSecondaryENI: true, + }) + } + + return ip +} + +// Validate host networking for the list of pods supplied +func ValidateHostNetworking(testType TestType, podValidationInputString string, nodeName string, f *framework.Framework) { + testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s", + podValidationInputString)} + + var shouldTestPodError bool + if NetworkingSetupSucceeds == testType { + testerArgs = append(testerArgs, "-test-setup=true") + } else if NetworkingSetupFails == testType { + testerArgs = append(testerArgs, "-test-setup=true") + shouldTestPodError = true + } else if NetworkingTearDownSucceeds == testType { + testerArgs = append(testerArgs, "-test-cleanup=true") + } else if NetworkingTearDownFails == testType { + testerArgs = append(testerArgs, "-test-cleanup=true") + shouldTestPodError = true + } + + testContainer := manifest.NewTestHelperContainer(). + Command([]string{"./networking"}). + Args(testerArgs). + Build() + + testPod := manifest.NewDefaultPodBuilder(). + Container(testContainer). + NodeName(nodeName). + HostNetwork(true). + Build() + + By("creating pod to test host networking setup") + testPod, err := f.K8sResourceManagers.PodManager(). + CreateAndWaitTillPodCompleted(testPod) + logs, errLogs := f.K8sResourceManagers.PodManager(). + PodLogs(testPod.Namespace, testPod.Name) + Expect(errLogs).ToNot(HaveOccurred()) + + fmt.Fprintln(GinkgoWriter, logs) + + if shouldTestPodError { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).ToNot(HaveOccurred()) + } + + By("deleting the host networking setup pod") + err = f.K8sResourceManagers.PodManager(). + DeleteAndWaitTillPodDeleted(testPod) + Expect(err).ToNot(HaveOccurred()) +} + +// GetPodsOnPrimaryAndSecondaryInterface returns the list of Pods on Primary Networking +// Interface and Secondary Network Interface on a given Node +func GetPodsOnPrimaryAndSecondaryInterface(node coreV1.Node, + podLabelKey string, podLabelVal string, f *framework.Framework) InterfaceTypeToPodList { + podList, err := f.K8sResourceManagers.PodManager(). + GetPodsWithLabelSelector(podLabelKey, podLabelVal) + Expect(err).ToNot(HaveOccurred()) + + instance, err := f.CloudServices.EC2(). + DescribeInstance(k8sUtils.GetInstanceIDFromNode(node)) + Expect(err).ToNot(HaveOccurred()) + + interfaceToPodList := InterfaceTypeToPodList{ + PodsOnPrimaryENI: []coreV1.Pod{}, + PodsOnSecondaryENI: []coreV1.Pod{}, + } + + ipToPod := map[string]coreV1.Pod{} + for _, pod := range podList.Items { + ipToPod[pod.Status.PodIP] = pod + } + + for _, nwInterface := range instance.NetworkInterfaces { + isPrimary := IsPrimaryENI(nwInterface, instance.PrivateIpAddress) + for _, ip := range nwInterface.PrivateIpAddresses { + if pod, found := ipToPod[*ip.PrivateIpAddress]; found { + if isPrimary { + interfaceToPodList.PodsOnPrimaryENI = + append(interfaceToPodList.PodsOnPrimaryENI, pod) + } else { + interfaceToPodList.PodsOnSecondaryENI = + append(interfaceToPodList.PodsOnSecondaryENI, pod) + } + } + } + } + return interfaceToPodList +} + +func IsPrimaryENI(nwInterface *ec2.InstanceNetworkInterface, instanceIPAddr *string) bool { + for _, privateIPAddress := range nwInterface.PrivateIpAddresses { + if *privateIPAddress.PrivateIpAddress == *instanceIPAddr { + return true + } + } + return false +} diff --git a/test/integration-new/ipamd/ipamd_suite_test.go b/test/integration-new/ipamd/ipamd_suite_test.go index cb82f3ee8d..ab79fd8f18 100644 --- a/test/integration-new/ipamd/ipamd_suite_test.go +++ b/test/integration-new/ipamd/ipamd_suite_test.go @@ -52,16 +52,4 @@ var _ = BeforeSuite(func() { instanceID := k8sUtils.GetInstanceIDFromNode(primaryNode) primaryInstance, err = f.CloudServices.EC2().DescribeInstance(instanceID) Expect(err).ToNot(HaveOccurred()) - - By("Delete coredns addon if it exists") - _, addonDeleteError = f.CloudServices.EKS().DeleteAddon("coredns", f.Options.ClusterName) - -}) - -var _ = AfterSuite(func() { - if addonDeleteError == nil { - By("Restore coredns addon") - _, err := f.CloudServices.EKS().CreateAddon("coredns", f.Options.ClusterName) - Expect(err).NotTo(HaveOccurred()) - } })