Snat tests: [agent is already updated] (aws#1513)

* resolved conflicts with go.sum

* Updated test agent image

* Removed redundant files

* Addressed PR comments

Fixed go.sum in root folder

Changed DescribeInstanceWithFilter to DescribeInstances
Moved GetPrimaryInstanceId from ec2 interface
Added GinkgoWriter

Updated Readme for Snat test

Rearranged snat_test logic
Updated Readme for test/e2e

* Minor change to logging

test/agent/go.sum

@@ -1,3 +1,5 @@
+github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
+github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=

test/e2e/README.md

@@ -7,6 +7,8 @@ The package contains e2e tests suites for `amazon-vpc-cni-k8s` .
   - No existing node group should be present the test creates new self managed node group with the reduced MAX_POD value.
 - Security Group For Pods Test
   - EKS Cluster should be v1.16+. This tests creates an additional Trunk ENI on all Nitro based instance present in the cluster. This could interfere with running integration test that test WARM_ENI_TARGET. For this reasons the test should either be run without any node group present in the cluster or at the very end.
+- Snat Test
+  - EKS Cluster should have atleast 1 private subnet and atleast 1 public subnet. These tests modifies the SNAT related variabls in aws-node, validates the IP table SNAT rules and checks for Internet Connectivity. 
 
 ####Testing
 Set the environment variables that will be passed to Ginkgo script. If you want to directly pass the arguments you can skip to next step.

test/e2e/snat/snat_suite_test.go

@@ -0,0 +1,165 @@
+package snat
+
+import (
+	"fmt"
+	"net/url"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils"
+	testUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+)
+
+var (
+	f                                                     *framework.Framework
+	props                                                 utils.NodeGroupProperties
+	primaryNodeInPublicSubnet, primaryNodeInPrivateSubnet v1.Node
+	privateSubnetId                                       string
+	input                                                 string
+)
+
+// Change this if you want to use your own Key Pair
+const DEFAULT_KEY_PAIR = "test-key-pair"
+
+func TestSnat(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Snat Suite")
+}
+
+var _ = BeforeSuite(func() {
+	f = framework.New(framework.GlobalOptions)
+
+	By("creating test namespace")
+	f.K8sResourceManagers.NamespaceManager().
+		CreateNamespace(testUtils.DefaultTestNamespace)
+
+	By("Getting existing nodes in the cluster")
+	nodes, err := f.K8sResourceManagers.NodeManager().GetAllNodes()
+	Expect(err).ToNot(HaveOccurred())
+
+	By("verifying more than 1 nodes are present for the test")
+	Expect(len(nodes.Items)).Should(BeNumerically(">", 1))
+
+	// Set the primary node for testing
+	primaryNodeInPublicSubnet = nodes.Items[0]
+
+	By("Getting Public and Private subnets")
+	vpcConfig, err := utils.GetClusterVPCConfig(f)
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(len(vpcConfig.PublicSubnetList)).To(BeNumerically(">", 0))
+	Expect(len(vpcConfig.PrivateSubnetList)).To(BeNumerically(">", 0))
+
+	msg := fmt.Sprintf("Creating a keyPair with name: %s if it doesn't exist", DEFAULT_KEY_PAIR)
+	By(msg)
+	keyPairOutput, _ := f.CloudServices.EC2().DescribeKey(DEFAULT_KEY_PAIR)
+
+	exists := false
+	if keyPairOutput != nil {
+		for _, keyPair := range keyPairOutput.KeyPairs {
+			if *keyPair.KeyName == DEFAULT_KEY_PAIR {
+				exists = true
+				break
+			}
+		}
+	}
+
+	if exists {
+		fmt.Fprintln(GinkgoWriter, "KeyPair already exists")
+	} else {
+		fmt.Fprintln(GinkgoWriter, "KeyPair doesn't exist, will be created")
+		_, err := f.CloudServices.EC2().CreateKey(DEFAULT_KEY_PAIR)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	privateSubnetId = vpcConfig.PrivateSubnetList[0]
+
+	By("Getting Cluster Security Group Id")
+	out, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName)
+	Expect(err).NotTo(HaveOccurred())
+
+	clusterSecurityGroupId := out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId
+
+	msg = fmt.Sprintf("Deploying a self managed nodegroup of size 1 in private subnet %s", privateSubnetId)
+	By(msg)
+	props = utils.NodeGroupProperties{
+		NgLabelKey:    "test-label-key",
+		NgLabelVal:    "test-label-val",
+		AsgSize:       1,
+		NodeGroupName: "snat-test-ng",
+		Subnet: []string{
+			privateSubnetId,
+		},
+		InstanceType: "m5.large",
+		KeyPairName:  DEFAULT_KEY_PAIR,
+	}
+
+	err = utils.CreateAndWaitTillSelfManagedNGReady(f, props)
+	Expect(err).NotTo(HaveOccurred())
+
+	nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(props.NgLabelKey,
+		props.NgLabelVal)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(nodeList.Items)).Should(BeNumerically(">", 0))
+
+	// Get ref to the only node from newly created nodegroup
+	primaryNodeInPrivateSubnet = nodeList.Items[0]
+	providerID := primaryNodeInPrivateSubnet.Spec.ProviderID
+	Expect(len(providerID)).To(BeNumerically(">", 0))
+
+	awsUrl, err := url.Parse(providerID)
+	Expect(err).NotTo(HaveOccurred())
+
+	instanceID := path.Base(awsUrl.Path)
+	Expect(len(instanceID)).To(BeNumerically(">", 0))
+
+	By("Fetching existing Security Groups from the newly created node group instance")
+
+	instance, err := f.CloudServices.EC2().DescribeInstance(instanceID)
+	Expect(err).NotTo(HaveOccurred())
+
+	existingSecurityGroups := instance.SecurityGroups
+	networkInterfaceId := getPrimaryNetworkInterfaceId(instance.NetworkInterfaces, instance.PrivateIpAddress)
+	Expect(networkInterfaceId).NotTo(Equal(BeNil()))
+
+	securityGroupIds := make([]*string, 0, len(existingSecurityGroups)+1)
+	for _, sg := range existingSecurityGroups {
+		securityGroupIds = append(securityGroupIds, sg.GroupId)
+	}
+	securityGroupIds = append(securityGroupIds, clusterSecurityGroupId)
+	By("Adding ClusterSecurityGroup to the new nodegroup Instance")
+	_, err = f.CloudServices.EC2().ModifyNetworkInterfaceSecurityGroups(securityGroupIds, networkInterfaceId)
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	//using default key pair created by test
+	if DEFAULT_KEY_PAIR == "test-key-pair" {
+		By("Deleting key-pair")
+		err := f.CloudServices.EC2().DeleteKey(DEFAULT_KEY_PAIR)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	By("Deleting test namespace")
+	f.K8sResourceManagers.NamespaceManager().
+		DeleteAndWaitTillNamespaceDeleted(testUtils.DefaultTestNamespace)
+
+	By("Deleting Managed Nodegroup")
+	err := utils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, props)
+	Expect(err).NotTo(HaveOccurred())
+})
+
+func getPrimaryNetworkInterfaceId(networkInterfaces []*ec2.InstanceNetworkInterface, instanceIPAddr *string) *string {
+	for _, ni := range networkInterfaces {
+		if strings.Compare(*ni.PrivateIpAddress, *instanceIPAddr) == 0 {
+			return ni.NetworkInterfaceId
+		}
+	}
+	return nil
+}

test/e2e/snat/snat_test.go

@@ -0,0 +1,170 @@
+package snat
+
+import (
+	"fmt"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
+	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"github.com/aws/aws-sdk-go/aws"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	TEST_POD_LABEL_KEY   = "test-pod-label-key"
+	TEST_POD_LABEL_VALUE = "test-pod-label-val"
+	EXTERNAL_DOMAIN      = "https://aws.amazon.com/"
+)
+
+var _ = Describe("SNAT tests", func() {
+	Context("ExternalSnat=false", func() {
+		BeforeEach(func() {
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
+				"AWS_VPC_K8S_CNI_EXTERNALSNAT": "false",
+			})
+		})
+
+		It("Pod in private subnet should have Internet access with External SNAT disabled", func() {
+			By("Checking External Domain Connectivity")
+			ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN)
+		})
+	})
+
+	Context("ExternSnat=true", func() {
+		BeforeEach(func() {
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
+				"AWS_VPC_K8S_CNI_EXTERNALSNAT": "true",
+			})
+		})
+
+		It("Pod in private subnet should have Internet access with External SNAT enabled", func() {
+			By("Checking External Domain Connectivity")
+			ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN)
+		})
+	})
+
+	Context("Validate AWS_VPC_K8S_CNI_RANDOMIZESNAT", func() {
+		It("Verify SNAT IP table rule by changing AWS_VPC_K8S_CNI_RANDOMIZESNAT", func() {
+			vpcOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(vpcOutput.Vpcs)).To(BeNumerically(">", 0))
+
+			numOfCidrs := len(vpcOutput.Vpcs[0].CidrBlockAssociationSet)
+
+			By("Check whether SNAT IP table has random-fully with AWS_VPC_K8S_CNI_RANDOMIZESNAT set to default value of prng")
+			ValidateIPTableRules("prng", numOfCidrs)
+
+			By("Setting AWS_VPC_K8S_CNI_RANDOMIZESNAT to none")
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
+				"AWS_VPC_K8S_CNI_RANDOMIZESNAT": "none",
+			})
+
+			By("Check where SNAT IP table rule is updated and it doesn't contain random port allocation")
+			ValidateIPTableRules("none", numOfCidrs)
+		})
+	})
+
+	Context("Validate AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS", func() {
+		It("Verify External Domain Connectivity by modifying AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS", func() {
+			By("Getting CIDR for primary node's private subnet")
+			out, err := f.CloudServices.EC2().DescribeSubnet(privateSubnetId)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(out.Subnets)).To(BeNumerically(">", 0))
+
+			cidrBlock := out.Subnets[0].CidrBlock
+			By("Updating AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS with private subnet CIDR")
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
+				"AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": aws.StringValue(cidrBlock),
+			})
+
+			By("Check External domain connectivity from this private subnet CIDR block")
+			ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN)
+		})
+	})
+
+	AfterEach(func() {
+		By("Reverting aws-node env variables to default values")
+		k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
+			"AWS_VPC_K8S_CNI_EXTERNALSNAT":  "false",
+			"AWS_VPC_K8S_CNI_RANDOMIZESNAT": "prng",
+		})
+		k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
+			"AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": {},
+		})
+	})
+})
+
+func ValidateExternalDomainConnectivity(url string) {
+	testerArgs := []string{
+		"-testExternalDomainConnectivity=true",
+		fmt.Sprintf("-url=%s", url),
+	}
+
+	testContainer := manifest.NewTestHelperContainer().
+		Command([]string{"./snat-utils"}).
+		Args(testerArgs).
+		Build()
+
+	testPodManifest := manifest.NewDefaultPodBuilder().
+		Container(testContainer).
+		NodeName(primaryNodeInPrivateSubnet.Name).
+		Name("snat-test-pod").
+		Build()
+
+	By("Deploying a test pod to check External domain access")
+	testPod, err := f.K8sResourceManagers.PodManager().
+		CreateAndWaitTillPodCompleted(testPodManifest)
+	Expect(err).NotTo(HaveOccurred())
+
+	logs, errLogs := f.K8sResourceManagers.PodManager().
+		PodLogs(testPod.Namespace, testPod.Name)
+	Expect(errLogs).ToNot(HaveOccurred())
+
+	fmt.Fprintln(GinkgoWriter, logs)
+
+	By("deleting the test pod")
+	err = f.K8sResourceManagers.PodManager().
+		DeleteAndWaitTillPodDeleted(testPod)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func ValidateIPTableRules(randomizedSNATValue string, numOfCidrs int) {
+	testerArgs := []string{
+		"-testIPTableRules=true",
+		fmt.Sprintf("-randomizedSNATValue=%s", randomizedSNATValue),
+		fmt.Sprintf("-numOfCidrs=%d", numOfCidrs),
+	}
+
+	hostNetworkContainer := manifest.NewTestHelperContainer().
+		Command([]string{"./snat-utils"}).
+		CapabilitiesForSecurityContext([]corev1.Capability{
+			"NET_ADMIN",
+		}, nil).
+		Args(testerArgs).
+		Build()
+
+	hostNetworkPodManifest := manifest.NewDefaultPodBuilder().
+		Container(hostNetworkContainer).
+		NodeName(primaryNodeInPublicSubnet.Name).
+		Name("host-network").
+		HostNetwork(true).
+		Build()
+
+	By("creating pod to check iptable SNAT rules on the host")
+	hostNetworkPod, err := f.K8sResourceManagers.PodManager().
+		CreateAndWaitTillPodCompleted(hostNetworkPodManifest)
+	Expect(err).NotTo(HaveOccurred())
+
+	logs, errLogs := f.K8sResourceManagers.PodManager().
+		PodLogs(hostNetworkPod.Namespace, hostNetworkPod.Name)
+	Expect(errLogs).ToNot(HaveOccurred())
+
+	fmt.Fprintln(GinkgoWriter, logs)
+
+	By("deleting the host networking setup pod")
+	err = f.K8sResourceManagers.PodManager().
+		DeleteAndWaitTillPodDeleted(hostNetworkPod)
+	Expect(err).ToNot(HaveOccurred())
+}