Skip to content

Commit

Permalink
Refactored code and Added cni addon upgrade/downgrade regression test (
Browse files Browse the repository at this point in the history
…#1861)

* Refactored code
Addon upgrade/downgrade test similar to #1795

Added tests for addon upgrade/downgrade

Changed DEFAULT version
Added addon status checks

Fetch latest addon version for given K8s Cluster

Update kops cluster config used in weekly tests (#1862)

* Change to kops cluster creation scripts

* Add logging for retry attempt

* Switch kops cluster to use docker container runtime

Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com>

Renamed package name for adddon tests

removed unnecessary changes
Fixed replica count for MTU and Veth test in host networking

Updated ENI/IP limits file for newly added instances (#1864)

* Added new instances

* Updated test readme

* needed rebase

* formatting

* remove all references to integration-new
migrate to ginkgo v2 in addon test files

* fix maxIPPerInterface count on pod_networking_suite

* Increase default deployment ready timeout

Co-authored-by: Vikas Basavaraj <5373156+vikasmb@users.noreply.github.com>
  • Loading branch information
cgchinmay and vikasmb authored May 17, 2022
1 parent e3cf980 commit 9d7bf47
Show file tree
Hide file tree
Showing 14 changed files with 537 additions and 201 deletions.
2 changes: 1 addition & 1 deletion scripts/run-cni-release-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ set -e

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
TEST_DIR="$SCRIPT_DIR/../test"
INTEGRATION_TEST_DIR="$TEST_DIR/integration-new"
INTEGRATION_TEST_DIR="$TEST_DIR/integration"
CALICO_TEST_DIR="$TEST_DIR/e2e/calico"

source "$SCRIPT_DIR"/lib/cluster.sh
Expand Down
2 changes: 1 addition & 1 deletion scripts/run-multus-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
set -e

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
INTEGRATION_TEST_DIR="$SCRIPT_DIR/../test/integration-new"
INTEGRATION_TEST_DIR="$SCRIPT_DIR/../test/integration"

source "$SCRIPT_DIR"/lib/common.sh
source "$SCRIPT_DIR"/lib/cluster.sh
Expand Down
4 changes: 4 additions & 0 deletions test/framework/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ type Options struct {
CalicoVersion string
ContainerRuntime string
InstanceType string
InitialAddon string
TargetAddon string
}

func (options *Options) BindFlags() {
Expand All @@ -47,6 +49,8 @@ func (options *Options) BindFlags() {
flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name")
flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name")
flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint")
flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied")
flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied")
flag.StringVar(&options.CalicoVersion, "calico-version", "3.22.0", "calico version to be tested")
flag.StringVar(&options.ContainerRuntime, "container-runtime", "", "Optionally can specify it as 'containerd' for the test nodes")
flag.StringVar(&options.InstanceType, "instance-type", "amd64", "Optionally specify instance type as arm64 for the test nodes")
Expand Down
57 changes: 49 additions & 8 deletions test/framework/resources/aws/services/eks.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,27 @@ import (

type EKS interface {
DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error)
CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error)
DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error)
CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error)
DescribeAddonVersions(AddonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error)
DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error)
DeleteAddon(AddOnInput *AddonInput) (*eks.DeleteAddonOutput, error)
GetLatestVersion(addonInput *AddonInput) (string, error)
}

type defaultEKS struct {
eksiface.EKSAPI
}

// Internal Addon Input struct
// subset of eks.AddonInput
// used by ginkgo tests
type AddonInput struct {
AddonName string
ClusterName string
AddonVersion string
K8sVersion string
}

func NewEKS(session *session.Session, endpoint string) EKS {
return &defaultEKS{
EKSAPI: eks.New(session, &aws.Config{
Expand All @@ -39,26 +52,54 @@ func NewEKS(session *session.Session, endpoint string) EKS {
}
}

func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) {
func (d defaultEKS) CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) {
createAddonInput := &eks.CreateAddonInput{
AddonName: aws.String(addon),
ClusterName: aws.String(clusterName),
AddonName: aws.String(addonInput.AddonName),
ClusterName: aws.String(addonInput.ClusterName),
}
if addonInput.AddonVersion != "" {
createAddonInput.SetAddonVersion(addonInput.AddonVersion)
createAddonInput.SetResolveConflicts("OVERWRITE")
}
return d.EKSAPI.CreateAddon(createAddonInput)
}

func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) {
func (d defaultEKS) DeleteAddon(addonInput *AddonInput) (*eks.DeleteAddonOutput, error) {
deleteAddonInput := &eks.DeleteAddonInput{
AddonName: aws.String(addon),
ClusterName: aws.String(clusterName),
AddonName: aws.String(addonInput.AddonName),
ClusterName: aws.String(addonInput.ClusterName),
}
return d.EKSAPI.DeleteAddon(deleteAddonInput)
}

func (d defaultEKS) DescribeAddonVersions(addonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) {
describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{
AddonName: aws.String(addonInput.AddonName),
KubernetesVersion: aws.String(addonInput.K8sVersion),
}
return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput)
}

func (d defaultEKS) DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) {
describeAddonInput := &eks.DescribeAddonInput{
AddonName: aws.String(addonInput.AddonName),
ClusterName: aws.String(addonInput.ClusterName),
}
return d.EKSAPI.DescribeAddon(describeAddonInput)
}

func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) {
describeClusterInput := &eks.DescribeClusterInput{
Name: aws.String(clusterName),
}

return d.EKSAPI.DescribeCluster(describeClusterInput)
}

func (d defaultEKS) GetLatestVersion(addonInput *AddonInput) (string, error) {
addonOutput, err := d.DescribeAddonVersions(addonInput)
if err != nil {
return "", err
}
return *addonOutput.Addons[0].AddonVersions[0].AddonVersion, nil
}
46 changes: 46 additions & 0 deletions test/framework/resources/k8s/utils/addon.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package utils

import (
"context"

"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/wait"
)

func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error {
ctx := context.Background()
return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
_, err := eks.DescribeAddon(&services.AddonInput{
AddonName: addonName,
ClusterName: clusterName,
})
if err != nil {
return false, err
}
return false, nil
}, ctx.Done())
}

func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error {
ctx := context.Background()
return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{
AddonName: addonName,
ClusterName: clusterName,
})
if err != nil {
return false, err
}

status := *describeAddonOutput.Addon.Status
if status == "CREATE_FAILED" || status == "DEGRADED" {
return false, errors.Errorf("Create Addon Failed, addon status: %s", status)
}
if status == "ACTIVE" {
return true, nil
}
return false, nil
}, ctx.Done())
}
2 changes: 1 addition & 1 deletion test/framework/utils/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,5 @@ const (
PollIntervalMedium = time.Second * 5
PollIntervalLong = time.Second * 20

DefaultDeploymentReadyTimeout = time.Second * 120
DefaultDeploymentReadyTimeout = time.Second * 300
)
4 changes: 2 additions & 2 deletions test/integration/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ all tests ran successfully in 0 minutes and 27 seconds
```

### Running release tests with scripts/run-cni-release-tests.sh
`run-cni-release-tests.sh` will run cni, ipamd, and cni-metrics-helper (integration tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/integration-new] and (Calico tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/e2e/calico]. The script _does not_ create a test cluster, instead it will run the test on cluster specified via variables required in the script. The tests are run on the vpc-cni version installed on the cluster(it does not upgrade/install any specific vpc-cni version). See script `update-cni-images.sh` to update the test cluster with required cni version before running the tests.
`run-cni-release-tests.sh` will run cni, ipamd, and cni-metrics-helper (integration tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/integration] and (Calico tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/e2e/calico]. The script _does not_ create a test cluster, instead it will run the test on cluster specified via variables required in the script. The tests are run on the vpc-cni version installed on the cluster(it does not upgrade/install any specific vpc-cni version). See script `update-cni-images.sh` to update the test cluster with required cni version before running the tests.

## Development of New Integration Tests

Expand Down Expand Up @@ -159,7 +159,7 @@ import (
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
//ginkgo and the assertion library: gomega are imported below
. "github.com/onsi/ginkgo"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
//v1 imported for Node libraries
v1 "k8s.io/api/core/v1"
Expand Down
84 changes: 84 additions & 0 deletions test/integration/addon-tests/cni_addon_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package addon_tests

import (
"fmt"
"testing"

"github.com/aws/amazon-vpc-cni-k8s/test/framework"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
v1 "k8s.io/api/core/v1"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

const InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type"

var f *framework.Framework
var maxIPPerInterface int
var primaryNode v1.Node
var secondaryNode v1.Node
var vpcCIDRs []string
var clusterVersion string
var latestAddonVersion string

func TestCNIPodNetworking(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "CNI Pod Networking Suite")
}

var _ = BeforeSuite(func() {
f = framework.New(framework.GlobalOptions)

By("creating test namespace")
f.K8sResourceManagers.NamespaceManager().
CreateNamespace(utils.DefaultTestNamespace)

By(fmt.Sprintf("getting the node with the node label key %s and value %s",
f.Options.NgNameLabelKey, f.Options.NgNameLabelVal))
nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)
Expect(err).ToNot(HaveOccurred())

By("verifying that atleast 1 nodes is present for the test")
Expect(len(nodes.Items)).Should(BeNumerically(">", 0))

// Set the primary and secondary node for testing
primaryNode = nodes.Items[0]
secondaryNode = nodes.Items[1]

By("getting the instance type from node label " + InstanceTypeNodeLabelKey)
instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey]

By("getting the network interface details from ec2")
instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType)
Expect(err).ToNot(HaveOccurred())

maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface)

By("describing the VPC to get the VPC CIDRs")
describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID)
Expect(err).ToNot(HaveOccurred())

for _, cidrBlockAssociationSet := range describeVPCOutput.Vpcs[0].CidrBlockAssociationSet {
vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock)
}

By("getting current cluster version")
clusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName)
Expect(err).NotTo(HaveOccurred())
clusterVersion = *clusterOutput.Cluster.Version

By("getting latest vpc-cni addon version")
latestAddonVersion, err = f.CloudServices.EKS().GetLatestVersion(&services.AddonInput{
AddonName: "vpc-cni",
K8sVersion: clusterVersion,
})
Expect(err).NotTo(HaveOccurred())
})

var _ = AfterSuite(func() {
By("deleting test namespace")
f.K8sResourceManagers.NamespaceManager().
DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
})
Loading

0 comments on commit 9d7bf47

Please sign in to comment.