Skip to content

Commit

Permalink
resolved conflicts
Browse files Browse the repository at this point in the history
Refactored vpc_cni_logfile_test

remove duplicate code

remove test agent changes from this PR

remove redundant Dockerfile changes
  • Loading branch information
Chinmay Gadgil committed Jul 19, 2021
1 parent 0af437a commit 3922fa1
Show file tree
Hide file tree
Showing 14 changed files with 694 additions and 161 deletions.
111 changes: 75 additions & 36 deletions go.sum

Large diffs are not rendered by default.

Binary file added test/agent/aws-node-config
Binary file not shown.
Binary file added test/agent/cni
Binary file not shown.
2 changes: 2 additions & 0 deletions test/agent/go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
Expand Down
153 changes: 153 additions & 0 deletions test/e2e/snat/snat_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package snat

import (
"fmt"
"testing"

"github.com/aws/amazon-vpc-cni-k8s/test/framework"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils"
testUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
"github.com/aws/aws-sdk-go/aws"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
)

var (
f *framework.Framework
props utils.NodeGroupProperties
primaryNodeInPublicSubnet, primaryNodeInPrivateSubnet v1.Node
privateSubnetId string
input string
)

// Change this if you want to use your own Key Pair
const DEFAULT_KEY_PAIR = "test-key-pair"

func TestSnat(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Snat Suite")
}

var _ = BeforeSuite(func() {
f = framework.New(framework.GlobalOptions)

By("creating test namespace")
f.K8sResourceManagers.NamespaceManager().
CreateNamespace(testUtils.DefaultTestNamespace)

By("Getting existing nodes in the cluster")
nodes, err := f.K8sResourceManagers.NodeManager().GetAllNodes()
Expect(err).ToNot(HaveOccurred())

By("verifying more than 1 nodes are present for the test")
Expect(len(nodes.Items)).Should(BeNumerically(">", 1))

// Set the primary node for testing
primaryNodeInPublicSubnet = nodes.Items[0]

By("Getting Public and Private subnets")
vpcConfig, err := utils.GetClusterVPCConfig(f)
Expect(err).ToNot(HaveOccurred())

Expect(len(vpcConfig.PublicSubnetList)).To(BeNumerically(">", 0))
Expect(len(vpcConfig.PrivateSubnetList)).To(BeNumerically(">", 0))

msg := fmt.Sprintf("Creating a keyPair with name: %s if it doesn't exist", DEFAULT_KEY_PAIR)
By(msg)
keyPairOutput, _ := f.CloudServices.EC2().DescribeKey(DEFAULT_KEY_PAIR)

exists := false
if keyPairOutput != nil {
for _, keyPair := range keyPairOutput.KeyPairs {
if *keyPair.KeyName == DEFAULT_KEY_PAIR {
exists = true
break
}
}
}

if exists {
fmt.Println("KeyPair already exists")
} else {
fmt.Println("KeyPair doesn't exist, will be created")
_, err := f.CloudServices.EC2().CreateKey(DEFAULT_KEY_PAIR)
Expect(err).NotTo(HaveOccurred())
}

privateSubnetId = vpcConfig.PrivateSubnetList[0]

By("Getting Cluster Security Group Id")
out, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName)
Expect(err).NotTo(HaveOccurred())

clusterSecurityGroupId := out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId

msg = fmt.Sprintf("Deploying a self managed nodegroup of size 1 in private subnet %s", privateSubnetId)
By(msg)
props = utils.NodeGroupProperties{
NgLabelKey: "test-label-key",
NgLabelVal: "test-label-val",
AsgSize: 1,
NodeGroupName: "snat-test-ng",
Subnet: []string{
privateSubnetId,
},
InstanceType: "m5.large",
KeyPairName: DEFAULT_KEY_PAIR,
}

err = utils.CreateAndWaitTillSelfManagedNGReady(f, props)
Expect(err).NotTo(HaveOccurred())

nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(props.NgLabelKey,
props.NgLabelVal)
Expect(err).ToNot(HaveOccurred())
Expect(len(nodeList.Items)).Should(BeNumerically(">", 0))

// Get ref to the only node from newly created nodegroup
primaryNodeInPrivateSubnet = nodeList.Items[0]

By("Fetching existing Security Groups from the newly created node group instance")
instanceOutput, err := f.CloudServices.EC2().DescribeInstancesWithFilters(map[*string][]*string{
aws.String("private-dns-name"): {
aws.String(primaryNodeInPrivateSubnet.Name),
},
})

Expect(err).NotTo(HaveOccurred())
Expect(len(instanceOutput.Reservations)).To(BeNumerically(">", 0))
Expect(len(instanceOutput.Reservations[0].Instances)).To(BeNumerically(">", 0))

instance := instanceOutput.Reservations[0].Instances[0]

existingSecurityGroups := instance.SecurityGroups
networkInterfaceId := f.CloudServices.EC2().GetPrimaryNetworkInterfaceId(instance.NetworkInterfaces, instance.PrivateIpAddress)
Expect(networkInterfaceId).NotTo(Equal(BeNil()))

securityGroupIds := make([]*string, 0, len(existingSecurityGroups)+1)
for _, sg := range existingSecurityGroups {
securityGroupIds = append(securityGroupIds, sg.GroupId)
}
securityGroupIds = append(securityGroupIds, clusterSecurityGroupId)
By("Adding ClusterSecurityGroup to the new nodegroup Instance")
_, err = f.CloudServices.EC2().ModifyNetworkInterfaceSecurityGroups(securityGroupIds, networkInterfaceId)
Expect(err).NotTo(HaveOccurred())
})

var _ = AfterSuite(func() {
//using default key pair created by test
if DEFAULT_KEY_PAIR == "test-key-pair" {
By("Deleting key-pair")
err := f.CloudServices.EC2().DeleteKey(DEFAULT_KEY_PAIR)
Expect(err).NotTo(HaveOccurred())
}

By("Deleting test namespace")
f.K8sResourceManagers.NamespaceManager().
DeleteAndWaitTillNamespaceDeleted(testUtils.DefaultTestNamespace)

By("Deleting Managed Nodegroup")
err := utils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, props)
Expect(err).NotTo(HaveOccurred())
})
160 changes: 160 additions & 0 deletions test/e2e/snat/snat_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
package snat

import (
"fmt"

"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
"github.com/aws/aws-sdk-go/aws"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
)

const (
TEST_POD_LABEL_KEY = "test-pod-label-key"
TEST_POD_LABEL_VALUE = "test-pod-label-val"
EXTERNAL_DOMAIN = "https://aws.amazon.com/"
)

var _ = Describe("SNAT test", func() {
It("Pod in private subnet should have Internet access with External SNAT disabled", func() {
By("Setting External SNAT to false")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
"AWS_VPC_K8S_CNI_EXTERNALSNAT": "false",
})

By("Checking External Domain Connectivity")
ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN)
})

It("Pod in private subnet should have Internet access with External SNAT enabled", func() {
By("Setting External SNAT to true")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
"AWS_VPC_K8S_CNI_EXTERNALSNAT": "true",
})

By("Checking External Domain Connectivity")
ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN)
})

It("Verify SNAT IP table rule by changing AWS_VPC_K8S_CNI_RANDOMIZESNAT", func() {
vpcOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID)
Expect(err).NotTo(HaveOccurred())
Expect(len(vpcOutput.Vpcs)).To(BeNumerically(">", 0))

numOfCidrs := len(vpcOutput.Vpcs[0].CidrBlockAssociationSet)

By("Check whether SNAT IP table has random-fully with AWS_VPC_K8S_CNI_RANDOMIZESNAT set to default value of prng")
ValidateIPTableRules("prng", numOfCidrs)

By("Setting AWS_VPC_K8S_CNI_RANDOMIZESNAT to none")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
"AWS_VPC_K8S_CNI_RANDOMIZESNAT": "none",
})

By("Check where SNAT IP table rule is updated and it doesn't contain random port allocation")
ValidateIPTableRules("none", numOfCidrs)
})

It("Verify External Domain Connectivity by modifying AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS", func() {
By("Getting CIDR for primary node's private subnet")
out, err := f.CloudServices.EC2().DescribeSubnet(privateSubnetId)
Expect(err).NotTo(HaveOccurred())
Expect(len(out.Subnets)).To(BeNumerically(">", 0))

cidrBlock := out.Subnets[0].CidrBlock
By("Updating AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS with private subnet CIDR")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
"AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": aws.StringValue(cidrBlock),
})

By("Check External domain connectivity from this private subnet CIDR block")
ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN)
})

AfterEach(func() {
By("Reverting aws-node env variables to default values")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
"AWS_VPC_K8S_CNI_EXTERNALSNAT": "false",
"AWS_VPC_K8S_CNI_RANDOMIZESNAT": "prng",
})
k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
"AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": {},
})
})
})

func ValidateExternalDomainConnectivity(url string) {
testerArgs := []string{
"-testExternalDomainConnectivity=true",
fmt.Sprintf("-url=%s", url),
}

testContainer := manifest.NewTestHelperContainer().
Command([]string{"./snat-utils"}).
Args(testerArgs).
Build()

testPodManifest := manifest.NewDefaultPodBuilder().
Container(testContainer).
NodeName(primaryNodeInPrivateSubnet.Name).
Name("snat-test-pod").
Build()

By("Deploying a test pod to check External domain access")
testPod, err := f.K8sResourceManagers.PodManager().
CreateAndWaitTillPodCompleted(testPodManifest)
Expect(err).NotTo(HaveOccurred())

logs, errLogs := f.K8sResourceManagers.PodManager().
PodLogs(testPod.Namespace, testPod.Name)
Expect(errLogs).ToNot(HaveOccurred())

fmt.Fprintln(GinkgoWriter, logs)

By("deleting the test pod")
err = f.K8sResourceManagers.PodManager().
DeleteAndWaitTillPodDeleted(testPod)
Expect(err).ToNot(HaveOccurred())
}

func ValidateIPTableRules(randomizedSNATValue string, numOfCidrs int) {
testerArgs := []string{
"-testIPTableRules=true",
fmt.Sprintf("-randomizedSNATValue=%s", randomizedSNATValue),
fmt.Sprintf("-numOfCidrs=%d", numOfCidrs),
}

hostNetworkContainer := manifest.NewTestHelperContainer().
Command([]string{"./snat-utils"}).
CapabilitiesForSecurityContext([]corev1.Capability{
"NET_ADMIN",
}, nil).
Args(testerArgs).
Build()

hostNetworkPodManifest := manifest.NewDefaultPodBuilder().
Container(hostNetworkContainer).
NodeName(primaryNodeInPublicSubnet.Name).
Name("host-network").
HostNetwork(true).
Build()

By("creating pod to check iptable SNAT rules on the host")
hostNetworkPod, err := f.K8sResourceManagers.PodManager().
CreateAndWaitTillPodCompleted(hostNetworkPodManifest)
Expect(err).NotTo(HaveOccurred())

logs, errLogs := f.K8sResourceManagers.PodManager().
PodLogs(hostNetworkPod.Namespace, hostNetworkPod.Name)
Expect(errLogs).ToNot(HaveOccurred())

fmt.Fprintln(GinkgoWriter, logs)

By("deleting the host networking setup pod")
err = f.K8sResourceManagers.PodManager().
DeleteAndWaitTillPodDeleted(hostNetworkPod)
Expect(err).ToNot(HaveOccurred())
}
Loading

0 comments on commit 3922fa1

Please sign in to comment.