Refactored code

Addon upgrade/downgrade test similar to aws#1795

Added tests for addon upgrade/downgrade

Changed DEFAULT version
Added addon status checks

Fetch latest addon version for given K8s Cluster

Update kops cluster config used in weekly tests (aws#1862)

* Change to kops cluster creation scripts

* Add logging for retry attempt

* Switch kops cluster to use docker container runtime

Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com>

Renamed package name for adddon tests

removed unnecessary changes
Fixed replica count for MTU and Veth test in host networking

Updated ENI/IP limits file for newly added instances (aws#1864)

* Added new instances

* Updated test readme

* needed rebase

* formatting

test/framework/options.go

@@ -34,6 +34,8 @@ type Options struct {
  NgNameLabelKey string
  NgNameLabelVal string
  EKSEndpoint string
+ InitialAddon string
+ TargetAddon string
 }
 
 func (options *Options) BindFlags() {
@@ -44,6 +46,8 @@ func (options *Options) BindFlags() {
  flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name")
  flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name")
  flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint")
+ flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied")
+ flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied")
 }
 
 func (options *Options) Validate() error {

test/framework/resources/aws/services/eks.go

@@ -22,14 +22,27 @@ import (
 
 type EKS interface {
  DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error)
- CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error)
- DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error)
+ CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error)
+ DescribeAddonVersions(AddonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error)
+ DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error)
+ DeleteAddon(AddOnInput *AddonInput) (*eks.DeleteAddonOutput, error)
+ GetLatestVersion(addonInput *AddonInput) (string, error)
 }
 
 type defaultEKS struct {
  eksiface.EKSAPI
 }
 
+// Internal Addon Input struct
+// subset of eks.AddonInput
+// used by ginkgo tests
+type AddonInput struct {
+ AddonName string
+ ClusterName string
+ AddonVersion string
+ K8sVersion string
+}
+
 func NewEKS(session *session.Session, endpoint string) EKS {
  return &defaultEKS{
  EKSAPI: eks.New(session, &aws.Config{
@@ -39,26 +52,54 @@ func NewEKS(session *session.Session, endpoint string) EKS {
  }
 }
 
-func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) {
+func (d defaultEKS) CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) {
  createAddonInput := &eks.CreateAddonInput{
- AddonName: aws.String(addon),
- ClusterName: aws.String(clusterName),
+ AddonName: aws.String(addonInput.AddonName),
+ ClusterName: aws.String(addonInput.ClusterName),
+ }
+ if addonInput.AddonVersion != "" {
+ createAddonInput.SetAddonVersion(addonInput.AddonVersion)
+ createAddonInput.SetResolveConflicts("OVERWRITE")
  }
  return d.EKSAPI.CreateAddon(createAddonInput)
 }
 
-func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) {
+func (d defaultEKS) DeleteAddon(addonInput *AddonInput) (*eks.DeleteAddonOutput, error) {
  deleteAddonInput := &eks.DeleteAddonInput{
- AddonName: aws.String(addon),
- ClusterName: aws.String(clusterName),
+ AddonName: aws.String(addonInput.AddonName),
+ ClusterName: aws.String(addonInput.ClusterName),
  }
  return d.EKSAPI.DeleteAddon(deleteAddonInput)
 }
 
+func (d defaultEKS) DescribeAddonVersions(addonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) {
+ describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{
+ AddonName: aws.String(addonInput.AddonName),
+ KubernetesVersion: aws.String(addonInput.K8sVersion),
+ }
+ return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput)
+}
+
+func (d defaultEKS) DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) {
+ describeAddonInput := &eks.DescribeAddonInput{
+ AddonName: aws.String(addonInput.AddonName),
+ ClusterName: aws.String(addonInput.ClusterName),
+ }
+ return d.EKSAPI.DescribeAddon(describeAddonInput)
+}
+
 func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) {
  describeClusterInput := &eks.DescribeClusterInput{
  Name: aws.String(clusterName),
  }
 
  return d.EKSAPI.DescribeCluster(describeClusterInput)
 }
+
+func (d defaultEKS) GetLatestVersion(addonInput *AddonInput) (string, error) {
+ addonOutput, err := d.DescribeAddonVersions(addonInput)
+ if err != nil {
+ return "", err
+ }
+ return *addonOutput.Addons[0].AddonVersions[0].AddonVersion, nil
+}

test/framework/resources/k8s/utils/addon.go

@@ -0,0 +1,46 @@
+package utils
+
+import (
+ "context"
+
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+ "github.com/pkg/errors"
+ "k8s.io/apimachinery/pkg/util/wait"
+)
+
+func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error {
+ ctx := context.Background()
+ return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+ _, err := eks.DescribeAddon(&services.AddonInput{
+ AddonName: addonName,
+ ClusterName: clusterName,
+ })
+ if err != nil {
+ return false, err
+ }
+ return false, nil
+ }, ctx.Done())
+}
+
+func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error {
+ ctx := context.Background()
+ return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+ describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{
+ AddonName: addonName,
+ ClusterName: clusterName,
+ })
+ if err != nil {
+ return false, err
+ }
+
+ status := *describeAddonOutput.Addon.Status
+ if status == "CREATE_FAILED" || status == "DEGRADED" {
+ return false, errors.Errorf("Create Addon Failed, addon status: %s", status)
+ }
+ if status == "ACTIVE" {
+ return true, nil
+ }
+ return false, nil
+ }, ctx.Done())
+}

test/integration-new/addon-tests/cni_addon_suite_test.go

@@ -0,0 +1,84 @@
+package addon_tests
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework"
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+ v1 "k8s.io/api/core/v1"
+
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+)
+
+const InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type"
+
+var f *framework.Framework
+var maxIPPerInterface int
+var primaryNode v1.Node
+var secondaryNode v1.Node
+var vpcCIDRs []string
+var clusterVersion string
+var latestAddonVersion string
+
+func TestCNIPodNetworking(t *testing.T) {
+ RegisterFailHandler(Fail)
+ RunSpecs(t, "CNI Pod Networking Suite")
+}
+
+var _ = BeforeSuite(func() {
+ f = framework.New(framework.GlobalOptions)
+
+ By("creating test namespace")
+ f.K8sResourceManagers.NamespaceManager().
+ CreateNamespace(utils.DefaultTestNamespace)
+
+ By(fmt.Sprintf("getting the node with the node label key %s and value %s",
+ f.Options.NgNameLabelKey, f.Options.NgNameLabelVal))
+ nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)
+ Expect(err).ToNot(HaveOccurred())
+
+ By("verifying that atleast 1 nodes is present for the test")
+ Expect(len(nodes.Items)).Should(BeNumerically(">", 0))
+
+ // Set the primary and secondary node for testing
+ primaryNode = nodes.Items[0]
+ secondaryNode = nodes.Items[1]
+
+ By("getting the instance type from node label " + InstanceTypeNodeLabelKey)
+ instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey]
+
+ By("getting the network interface details from ec2")
+ instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType)
+ Expect(err).ToNot(HaveOccurred())
+
+ maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface)
+
+ By("describing the VPC to get the VPC CIDRs")
+ describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID)
+ Expect(err).ToNot(HaveOccurred())
+
+ for _, cidrBlockAssociationSet := range describeVPCOutput.Vpcs[0].CidrBlockAssociationSet {
+ vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock)
+ }
+
+ By("getting current cluster version")
+ clusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName)
+ Expect(err).NotTo(HaveOccurred())
+ clusterVersion = *clusterOutput.Cluster.Version
+
+ By("getting latest vpc-cni addon version")
+ latestAddonVersion, err = f.CloudServices.EKS().GetLatestVersion(&services.AddonInput{
+ AddonName: "vpc-cni",
+ K8sVersion: clusterVersion,
+ })
+ Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+ By("deleting test namespace")
+ f.K8sResourceManagers.NamespaceManager().
+ DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
+})

test/integration-new/addon-tests/cni_addon_test.go

@@ -0,0 +1,154 @@
+package addon_tests
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
+ k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+ "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+ "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common"
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+ v1 "k8s.io/api/apps/v1"
+)
+
+var (
+ err error
+ podLabelKey = "app"
+ podLabelVal = "host-networking-test"
+ deployment *v1.Deployment
+)
+
+const (
+ // Using v1.9.x as the default since v1.7.5 has following issue:https://github.com/aws/amazon-vpc-cni-k8s/pull/1341
+ DEFAULT_VERSION = "v1.9.1-eksbuild.1"
+)
+
+var _ = Describe("cni addon upgrade/downgrade test", func() {
+ Context("test host networking", func() {
+ It("create test deployment with initial addon and verify deletion with target addon", func() {
+ targetAddonVersion := f.Options.TargetAddon
+ initialAddonVersion := f.Options.InitialAddon
+ if len(f.Options.TargetAddon) == 0 {
+ targetAddonVersion = latestAddonVersion
+ }
+ By(fmt.Sprintf("using target addon version as %s", targetAddonVersion))
+
+ if len(f.Options.InitialAddon) == 0 {
+ initialAddonVersion = DEFAULT_VERSION
+ }
+ By(fmt.Sprintf("using initial addon version as %s", initialAddonVersion))
+
+ ApplyAddOn(initialAddonVersion)
+
+ interfaceTypeToPodList := getTestPodList()
+ By("generating the pod networking validation input to be passed to tester")
+ input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize()
+ Expect(err).NotTo(HaveOccurred())
+
+ By("validating host networking is setup correctly with initial addon")
+ common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)
+
+ By("upgrading to target addon version")
+ ApplyAddOn(targetAddonVersion)
+
+ deleteTestDeployment()
+
+ By("validating host networking is teared down correctly with target addon")
+ common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
+ })
+
+ It("create test deployment with target addon and verify deletion with initial addon", func() {
+ targetAddonVersion := f.Options.TargetAddon
+ initialAddonVersion := f.Options.InitialAddon
+ if len(f.Options.TargetAddon) == 0 {
+ targetAddonVersion = latestAddonVersion
+ }
+ By(fmt.Sprintf("using target addon version as %s", targetAddonVersion))
+
+ if len(f.Options.InitialAddon) == 0 {
+ initialAddonVersion = DEFAULT_VERSION
+ }
+ By(fmt.Sprintf("using initial addon version as %s", initialAddonVersion))
+
+ ApplyAddOn(targetAddonVersion)
+
+ interfaceTypeToPodList := getTestPodList()
+ By("generating the pod networking validation input to be passed to tester")
+ input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize()
+ Expect(err).NotTo(HaveOccurred())
+
+ By("validating host networking is setup correctly with target addon")
+ common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)
+
+ By("downgrading to initial addon version")
+ ApplyAddOn(initialAddonVersion)
+
+ deleteTestDeployment()
+
+ By("validating host networking is teared down correctly with initial addon")
+ common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
+ })
+ })
+})
+
+func deleteTestDeployment() {
+ By("deleting the deployment to test teardown")
+ err = f.K8sResourceManagers.DeploymentManager().
+ DeleteAndWaitTillDeploymentIsDeleted(deployment)
+ Expect(err).ToNot(HaveOccurred())
+
+ By("waiting to allow CNI to tear down networking for terminated pods")
+ time.Sleep(time.Second * 60)
+}
+
+func getTestPodList() common.InterfaceTypeToPodList {
+ deployment = manifest.NewBusyBoxDeploymentBuilder().
+ Replicas(maxIPPerInterface*2).
+ PodLabel(podLabelKey, podLabelVal).
+ NodeName(primaryNode.Name).
+ Build()
+
+ By("creating a deployment to launch pod using primary and secondary ENI IP")
+ _, err := f.K8sResourceManagers.DeploymentManager().
+ CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
+ Expect(err).ToNot(HaveOccurred())
+
+ By("getting the list of pods using IP from primary and secondary ENI")
+ interfaceTypeToPodList :=
+ common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)
+
+ return interfaceTypeToPodList
+}
+
+func ApplyAddOn(version string) {
+ By("delete existing cni addon if any")
+ _, err := f.CloudServices.EKS().DescribeAddon(&services.AddonInput{
+ AddonName: "vpc-cni",
+ ClusterName: f.Options.ClusterName,
+ })
+ if err == nil {
+ _, err := f.CloudServices.EKS().DeleteAddon(&services.AddonInput{
+ AddonName: "vpc-cni",
+ ClusterName: f.Options.ClusterName,
+ })
+ Expect(err).NotTo(HaveOccurred())
+
+ By("wait till the old addon is deleted")
+ err = k8sUtils.WaitTillAddonIsDeleted(f.CloudServices.EKS(), "vpc-cni", f.Options.ClusterName)
+ Expect(err).To(HaveOccurred())
+ }
+ By(fmt.Sprintf("install the new addon with version: %s", version))
+ _, err = f.CloudServices.EKS().CreateAddon(&services.AddonInput{
+ AddonName: "vpc-cni",
+ ClusterName: f.Options.ClusterName,
+ AddonVersion: version,
+ })
+ Expect(err).NotTo(HaveOccurred())
+
+ By("wait till the addon is active")
+ err = k8sUtils.WaitTillAddonIsActive(f.CloudServices.EKS(), "vpc-cni", f.Options.ClusterName)
+ Expect(err).NotTo(HaveOccurred())
+}