Skip to content

Commit

Permalink
Refactored code
Browse files Browse the repository at this point in the history 
Addon upgrade/downgrade test similar to aws#1795

Added tests for addon upgrade/downgrade

Changed DEFAULT version
Added addon status checks

Fetch latest addon version for given K8s Cluster

Update kops cluster config used in weekly tests (aws#1862)

* Change to kops cluster creation scripts

* Add logging for retry attempt

* Switch kops cluster to use docker container runtime

Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com>
  • Loading branch information
@jayanthvn
Chinmay Gadgil and jayanthvn committed Feb 15, 2022
1 parent 5447794 commit 1eff676
Show file tree
Hide file tree
Showing 8 changed files with 524 additions and 182 deletions.
4 changes: 4 additions & 0 deletions test/framework/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ type Options struct {
NgNameLabelKey string
NgNameLabelVal string
EKSEndpoint string
InitialAddon string
TargetAddon string
}

func (options *Options) BindFlags() {
Expand All @@ -44,6 +46,8 @@ func (options *Options) BindFlags() {
flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name")
flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name")
flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint")
flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied")
flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied")
}

func (options *Options) Validate() error {
Expand Down
62 changes: 54 additions & 8 deletions test/framework/resources/aws/services/eks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,32 @@ import (

type EKS interface {
DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error)
CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error)
DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error)
CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error)
DescribeAddonVersions(AddonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error)
DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error)
DeleteAddon(AddOnInput *AddonInput) (*eks.DeleteAddonOutput, error)
GetLatestVersion(addonInput *AddonInput) (string, error)
}

type defaultEKS struct {
eksiface.EKSAPI
}

// Internal Addon Input struct
// subset of eks.AddonInput
// used by ginkgo tests
type AddonInput struct {
AddonName string
ClusterName string
AddonVersion string
K8sVersion string
}

type VersionWithBuild struct {
version uint
build uint
}

func NewEKS(session *session.Session, endpoint string) EKS {
return &defaultEKS{
EKSAPI: eks.New(session, &aws.Config{
Expand All @@ -39,26 +57,54 @@ func NewEKS(session *session.Session, endpoint string) EKS {
}
}

func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) {
func (d defaultEKS) CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) {
createAddonInput := &eks.CreateAddonInput{
AddonName: aws.String(addon),
ClusterName: aws.String(clusterName),
AddonName: aws.String(addonInput.AddonName),
ClusterName: aws.String(addonInput.ClusterName),
}
if addonInput.AddonVersion != "" {
createAddonInput.SetAddonVersion(addonInput.AddonVersion)
createAddonInput.SetResolveConflicts("OVERWRITE")
}
return d.EKSAPI.CreateAddon(createAddonInput)
}

func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) {
func (d defaultEKS) DeleteAddon(addonInput *AddonInput) (*eks.DeleteAddonOutput, error) {
deleteAddonInput := &eks.DeleteAddonInput{
AddonName: aws.String(addon),
ClusterName: aws.String(clusterName),
AddonName: aws.String(addonInput.AddonName),
ClusterName: aws.String(addonInput.ClusterName),
}
return d.EKSAPI.DeleteAddon(deleteAddonInput)
}

func (d defaultEKS) DescribeAddonVersions(addonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) {
describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{
AddonName: aws.String(addonInput.AddonName),
KubernetesVersion: aws.String(addonInput.K8sVersion),
}
return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput)
}

func (d defaultEKS) DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) {
describeAddonInput := &eks.DescribeAddonInput{
AddonName: aws.String(addonInput.AddonName),
ClusterName: aws.String(addonInput.ClusterName),
}
return d.EKSAPI.DescribeAddon(describeAddonInput)
}

func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) {
describeClusterInput := &eks.DescribeClusterInput{
Name: aws.String(clusterName),
}

return d.EKSAPI.DescribeCluster(describeClusterInput)
}

func (d defaultEKS) GetLatestVersion(addonInput *AddonInput) (string, error) {
addonOutput, err := d.DescribeAddonVersions(addonInput)
if err != nil {
return "", err
}
return *addonOutput.Addons[0].AddonVersions[0].AddonVersion, nil
}
46 changes: 46 additions & 0 deletions test/framework/resources/k8s/utils/addon.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package utils

import (
"context"

"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/wait"
)

func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error {
ctx := context.Background()
return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
_, err := eks.DescribeAddon(&services.AddonInput{
AddonName: addonName,
ClusterName: clusterName,
})
if err != nil {
return false, err
}
return false, nil
}, ctx.Done())
}

func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error {
ctx := context.Background()
return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{
AddonName: addonName,
ClusterName: clusterName,
})
if err != nil {
return false, err
}

status := *describeAddonOutput.Addon.Status
if status == "CREATE_FAILED" || status == "DEGRADED" {
return false, errors.Errorf("Create Addon Failed, addon status: %s", status)
}
if status == "ACTIVE" {
return true, nil
}
return false, nil
}, ctx.Done())
}
132 changes: 19 additions & 113 deletions test/integration-new/cni/host_networking_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,32 +14,21 @@
package cni

import (
"fmt"
"strconv"
"time"

"github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input"
v1 "k8s.io/api/core/v1"

"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
"github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

// TODO: Instead of passing the list of pods to the test helper, have the test helper get the pod on node

type TestType int

const (
NetworkingTearDownSucceeds TestType = iota
NetworkingTearDownFails
NetworkingSetupSucceeds
NetworkingSetupFails
)

const (
AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU"
AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX"
Expand Down Expand Up @@ -76,8 +65,7 @@ var _ = Describe("test host networking", func() {
Expect(err).ToNot(HaveOccurred())

By("getting the list of pods using IP from primary and secondary ENI")
interfaceTypeToPodList :=
GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal)
interfaceTypeToPodList := common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)

// Primary ENI and Secondary ENI IPs are handled differently when setting up
// the host networking rule hence this check
Expand All @@ -87,11 +75,11 @@ var _ = Describe("test host networking", func() {
Should(BeNumerically(">", 0))

By("generating the pod networking validation input to be passed to tester")
input, err := GetPodNetworkingValidationInput(interfaceTypeToPodList).Serialize()
input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize()
Expect(err).NotTo(HaveOccurred())

By("validating host networking setup is setup correctly")
ValidateHostNetworking(NetworkingSetupSucceeds, input)
common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)

By("deleting the deployment to test teardown")
err = f.K8sResourceManagers.DeploymentManager().
Expand All @@ -102,7 +90,7 @@ var _ = Describe("test host networking", func() {
time.Sleep(time.Second * 60)

By("validating host networking is teared down correctly")
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
})

It("Validate Host Networking setup after changing MTU and Veth Prefix", func() {
Expand All @@ -125,18 +113,18 @@ var _ = Describe("test host networking", func() {

By("getting the list of pods using IP from primary and secondary ENI")
interfaceTypeToPodList :=
GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal)
common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)

By("generating the pod networking validation input to be passed to tester")
podNetworkingValidationInput := GetPodNetworkingValidationInput(interfaceTypeToPodList)
podNetworkingValidationInput := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs)
podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
podNetworkingValidationInput.ValidateMTU = true
podNetworkingValidationInput.MTU = NEW_MTU_VAL
input, err := podNetworkingValidationInput.Serialize()
Expect(err).NotTo(HaveOccurred())

By("validating host networking setup is setup correctly with MTU check as well")
ValidateHostNetworking(NetworkingSetupSucceeds, input)
common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)

By("deleting the deployment to test teardown")
err = f.K8sResourceManagers.DeploymentManager().
Expand All @@ -147,7 +135,7 @@ var _ = Describe("test host networking", func() {
time.Sleep(time.Second * 60)

By("validating host networking is teared down correctly")
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
})
})

Expand All @@ -165,39 +153,39 @@ var _ = Describe("test host networking", func() {
CreatAndWaitTillRunning(parkingPod)
Expect(err).ToNot(HaveOccurred())

validInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{
validInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{
PodsOnPrimaryENI: []v1.Pod{*parkingPod},
}).Serialize()
}, vpcCIDRs).Serialize()
Expect(err).NotTo(HaveOccurred())

By("first validating the tester work on valid input")
ValidateHostNetworking(NetworkingSetupSucceeds, validInput)
common.ValidateHostNetworking(common.NetworkingSetupSucceeds, validInput, primaryNode.Name, f)

By("validating tester fails when invalid IP is passed")
invalidPod := parkingPod.DeepCopy()
invalidPod.Status.PodIP = "1.1.1.1"

invalidInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{
invalidInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{
PodsOnPrimaryENI: []v1.Pod{*invalidPod},
}).Serialize()
}, vpcCIDRs).Serialize()
Expect(err).NotTo(HaveOccurred())

ValidateHostNetworking(NetworkingSetupFails, invalidInput)
common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f)

By("validating the tester fails when invalid namespace is passed")
invalidPod = parkingPod.DeepCopy()
// veth pair name is generated using namespace+name so the test should fail
invalidPod.Namespace = "different"

invalidInput, err = GetPodNetworkingValidationInput(InterfaceTypeToPodList{
invalidInput, err = common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{
PodsOnPrimaryENI: []v1.Pod{*invalidPod},
}).Serialize()
}, vpcCIDRs).Serialize()
Expect(err).NotTo(HaveOccurred())

ValidateHostNetworking(NetworkingSetupFails, invalidInput)
common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f)

By("validating the tester fails when tear down check is run on running pod")
ValidateHostNetworking(NetworkingTearDownFails, validInput)
common.ValidateHostNetworking(common.NetworkingTearDownFails, validInput, primaryNode.Name, f)

By("deleting the parking pod")
err = f.K8sResourceManagers.PodManager().
Expand All @@ -206,85 +194,3 @@ var _ = Describe("test host networking", func() {
})
})
})

// Validate host networking for the list of pods supplied
func ValidateHostNetworking(testType TestType, podValidationInputString string) {
testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s",
podValidationInputString)}

var shouldTestPodError bool
if NetworkingSetupSucceeds == testType {
testerArgs = append(testerArgs, "-test-setup=true")
} else if NetworkingSetupFails == testType {
testerArgs = append(testerArgs, "-test-setup=true")
shouldTestPodError = true
} else if NetworkingTearDownSucceeds == testType {
testerArgs = append(testerArgs, "-test-cleanup=true")
} else if NetworkingTearDownFails == testType {
testerArgs = append(testerArgs, "-test-cleanup=true")
shouldTestPodError = true
}

testContainer := manifest.NewTestHelperContainer().
Command([]string{"./networking"}).
Args(testerArgs).
Build()

testPod := manifest.NewDefaultPodBuilder().
Container(testContainer).
NodeName(primaryNode.Name).
HostNetwork(true).
Build()

By("creating pod to test host networking setup")
testPod, err := f.K8sResourceManagers.PodManager().
CreateAndWaitTillPodCompleted(testPod)
logs, errLogs := f.K8sResourceManagers.PodManager().
PodLogs(testPod.Namespace, testPod.Name)
Expect(errLogs).ToNot(HaveOccurred())

fmt.Fprintln(GinkgoWriter, logs)

if shouldTestPodError {
Expect(err).To(HaveOccurred())
} else {
Expect(err).ToNot(HaveOccurred())
}

By("deleting the host networking setup pod")
err = f.K8sResourceManagers.PodManager().
DeleteAndWaitTillPodDeleted(testPod)
Expect(err).ToNot(HaveOccurred())
}

// GetPodNetworkingValidationInput returns input string containing the list of pods for which
// the host networking has to be tested
func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) input.PodNetworkingValidationInput {

ip := input.PodNetworkingValidationInput{
VPCCidrRange: vpcCIDRs,
VethPrefix: "eni",
PodList: []input.Pod{},
ValidateMTU: false,
}

for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI {
ip.PodList = append(ip.PodList, input.Pod{
PodName: primaryENIPod.Name,
PodNamespace: primaryENIPod.Namespace,
PodIPv4Address: primaryENIPod.Status.PodIP,
IsIPFromSecondaryENI: false,
})
}

for _, secondaryENIPod := range interfaceTypeToPodList.PodsOnSecondaryENI {
ip.PodList = append(ip.PodList, input.Pod{
PodName: secondaryENIPod.Name,
PodNamespace: secondaryENIPod.Namespace,
PodIPv4Address: secondaryENIPod.Status.PodIP,
IsIPFromSecondaryENI: true,
})
}

return ip
}
Loading

0 comments on commit 1eff676

Please sign in to comment.