-
Notifications
You must be signed in to change notification settings - Fork 744
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactored vpc_cni_logfile_test remove duplicate code remove test agent changes from this PR remove redundant Dockerfile changes
- Loading branch information
Chinmay Gadgil
committed
Jul 19, 2021
1 parent
0af437a
commit 3922fa1
Showing
14 changed files
with
694 additions
and
161 deletions.
There are no files selected for viewing
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,153 @@ | ||
package snat | ||
|
||
import ( | ||
"fmt" | ||
"testing" | ||
|
||
"github.com/aws/amazon-vpc-cni-k8s/test/framework" | ||
"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils" | ||
testUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
"github.com/aws/aws-sdk-go/aws" | ||
. "github.com/onsi/ginkgo" | ||
. "github.com/onsi/gomega" | ||
v1 "k8s.io/api/core/v1" | ||
) | ||
|
||
var ( | ||
f *framework.Framework | ||
props utils.NodeGroupProperties | ||
primaryNodeInPublicSubnet, primaryNodeInPrivateSubnet v1.Node | ||
privateSubnetId string | ||
input string | ||
) | ||
|
||
// Change this if you want to use your own Key Pair | ||
const DEFAULT_KEY_PAIR = "test-key-pair" | ||
|
||
func TestSnat(t *testing.T) { | ||
RegisterFailHandler(Fail) | ||
RunSpecs(t, "Snat Suite") | ||
} | ||
|
||
var _ = BeforeSuite(func() { | ||
f = framework.New(framework.GlobalOptions) | ||
|
||
By("creating test namespace") | ||
f.K8sResourceManagers.NamespaceManager(). | ||
CreateNamespace(testUtils.DefaultTestNamespace) | ||
|
||
By("Getting existing nodes in the cluster") | ||
nodes, err := f.K8sResourceManagers.NodeManager().GetAllNodes() | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
By("verifying more than 1 nodes are present for the test") | ||
Expect(len(nodes.Items)).Should(BeNumerically(">", 1)) | ||
|
||
// Set the primary node for testing | ||
primaryNodeInPublicSubnet = nodes.Items[0] | ||
|
||
By("Getting Public and Private subnets") | ||
vpcConfig, err := utils.GetClusterVPCConfig(f) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
Expect(len(vpcConfig.PublicSubnetList)).To(BeNumerically(">", 0)) | ||
Expect(len(vpcConfig.PrivateSubnetList)).To(BeNumerically(">", 0)) | ||
|
||
msg := fmt.Sprintf("Creating a keyPair with name: %s if it doesn't exist", DEFAULT_KEY_PAIR) | ||
By(msg) | ||
keyPairOutput, _ := f.CloudServices.EC2().DescribeKey(DEFAULT_KEY_PAIR) | ||
|
||
exists := false | ||
if keyPairOutput != nil { | ||
for _, keyPair := range keyPairOutput.KeyPairs { | ||
if *keyPair.KeyName == DEFAULT_KEY_PAIR { | ||
exists = true | ||
break | ||
} | ||
} | ||
} | ||
|
||
if exists { | ||
fmt.Println("KeyPair already exists") | ||
} else { | ||
fmt.Println("KeyPair doesn't exist, will be created") | ||
_, err := f.CloudServices.EC2().CreateKey(DEFAULT_KEY_PAIR) | ||
Expect(err).NotTo(HaveOccurred()) | ||
} | ||
|
||
privateSubnetId = vpcConfig.PrivateSubnetList[0] | ||
|
||
By("Getting Cluster Security Group Id") | ||
out, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) | ||
Expect(err).NotTo(HaveOccurred()) | ||
|
||
clusterSecurityGroupId := out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId | ||
|
||
msg = fmt.Sprintf("Deploying a self managed nodegroup of size 1 in private subnet %s", privateSubnetId) | ||
By(msg) | ||
props = utils.NodeGroupProperties{ | ||
NgLabelKey: "test-label-key", | ||
NgLabelVal: "test-label-val", | ||
AsgSize: 1, | ||
NodeGroupName: "snat-test-ng", | ||
Subnet: []string{ | ||
privateSubnetId, | ||
}, | ||
InstanceType: "m5.large", | ||
KeyPairName: DEFAULT_KEY_PAIR, | ||
} | ||
|
||
err = utils.CreateAndWaitTillSelfManagedNGReady(f, props) | ||
Expect(err).NotTo(HaveOccurred()) | ||
|
||
nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(props.NgLabelKey, | ||
props.NgLabelVal) | ||
Expect(err).ToNot(HaveOccurred()) | ||
Expect(len(nodeList.Items)).Should(BeNumerically(">", 0)) | ||
|
||
// Get ref to the only node from newly created nodegroup | ||
primaryNodeInPrivateSubnet = nodeList.Items[0] | ||
|
||
By("Fetching existing Security Groups from the newly created node group instance") | ||
instanceOutput, err := f.CloudServices.EC2().DescribeInstancesWithFilters(map[*string][]*string{ | ||
aws.String("private-dns-name"): { | ||
aws.String(primaryNodeInPrivateSubnet.Name), | ||
}, | ||
}) | ||
|
||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(len(instanceOutput.Reservations)).To(BeNumerically(">", 0)) | ||
Expect(len(instanceOutput.Reservations[0].Instances)).To(BeNumerically(">", 0)) | ||
|
||
instance := instanceOutput.Reservations[0].Instances[0] | ||
|
||
existingSecurityGroups := instance.SecurityGroups | ||
networkInterfaceId := f.CloudServices.EC2().GetPrimaryNetworkInterfaceId(instance.NetworkInterfaces, instance.PrivateIpAddress) | ||
Expect(networkInterfaceId).NotTo(Equal(BeNil())) | ||
|
||
securityGroupIds := make([]*string, 0, len(existingSecurityGroups)+1) | ||
for _, sg := range existingSecurityGroups { | ||
securityGroupIds = append(securityGroupIds, sg.GroupId) | ||
} | ||
securityGroupIds = append(securityGroupIds, clusterSecurityGroupId) | ||
By("Adding ClusterSecurityGroup to the new nodegroup Instance") | ||
_, err = f.CloudServices.EC2().ModifyNetworkInterfaceSecurityGroups(securityGroupIds, networkInterfaceId) | ||
Expect(err).NotTo(HaveOccurred()) | ||
}) | ||
|
||
var _ = AfterSuite(func() { | ||
//using default key pair created by test | ||
if DEFAULT_KEY_PAIR == "test-key-pair" { | ||
By("Deleting key-pair") | ||
err := f.CloudServices.EC2().DeleteKey(DEFAULT_KEY_PAIR) | ||
Expect(err).NotTo(HaveOccurred()) | ||
} | ||
|
||
By("Deleting test namespace") | ||
f.K8sResourceManagers.NamespaceManager(). | ||
DeleteAndWaitTillNamespaceDeleted(testUtils.DefaultTestNamespace) | ||
|
||
By("Deleting Managed Nodegroup") | ||
err := utils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, props) | ||
Expect(err).NotTo(HaveOccurred()) | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,160 @@ | ||
package snat | ||
|
||
import ( | ||
"fmt" | ||
|
||
"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" | ||
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" | ||
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" | ||
"github.com/aws/aws-sdk-go/aws" | ||
. "github.com/onsi/ginkgo" | ||
. "github.com/onsi/gomega" | ||
corev1 "k8s.io/api/core/v1" | ||
) | ||
|
||
const ( | ||
TEST_POD_LABEL_KEY = "test-pod-label-key" | ||
TEST_POD_LABEL_VALUE = "test-pod-label-val" | ||
EXTERNAL_DOMAIN = "https://aws.amazon.com/" | ||
) | ||
|
||
var _ = Describe("SNAT test", func() { | ||
It("Pod in private subnet should have Internet access with External SNAT disabled", func() { | ||
By("Setting External SNAT to false") | ||
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ | ||
"AWS_VPC_K8S_CNI_EXTERNALSNAT": "false", | ||
}) | ||
|
||
By("Checking External Domain Connectivity") | ||
ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN) | ||
}) | ||
|
||
It("Pod in private subnet should have Internet access with External SNAT enabled", func() { | ||
By("Setting External SNAT to true") | ||
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ | ||
"AWS_VPC_K8S_CNI_EXTERNALSNAT": "true", | ||
}) | ||
|
||
By("Checking External Domain Connectivity") | ||
ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN) | ||
}) | ||
|
||
It("Verify SNAT IP table rule by changing AWS_VPC_K8S_CNI_RANDOMIZESNAT", func() { | ||
vpcOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(len(vpcOutput.Vpcs)).To(BeNumerically(">", 0)) | ||
|
||
numOfCidrs := len(vpcOutput.Vpcs[0].CidrBlockAssociationSet) | ||
|
||
By("Check whether SNAT IP table has random-fully with AWS_VPC_K8S_CNI_RANDOMIZESNAT set to default value of prng") | ||
ValidateIPTableRules("prng", numOfCidrs) | ||
|
||
By("Setting AWS_VPC_K8S_CNI_RANDOMIZESNAT to none") | ||
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ | ||
"AWS_VPC_K8S_CNI_RANDOMIZESNAT": "none", | ||
}) | ||
|
||
By("Check where SNAT IP table rule is updated and it doesn't contain random port allocation") | ||
ValidateIPTableRules("none", numOfCidrs) | ||
}) | ||
|
||
It("Verify External Domain Connectivity by modifying AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS", func() { | ||
By("Getting CIDR for primary node's private subnet") | ||
out, err := f.CloudServices.EC2().DescribeSubnet(privateSubnetId) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(len(out.Subnets)).To(BeNumerically(">", 0)) | ||
|
||
cidrBlock := out.Subnets[0].CidrBlock | ||
By("Updating AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS with private subnet CIDR") | ||
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ | ||
"AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": aws.StringValue(cidrBlock), | ||
}) | ||
|
||
By("Check External domain connectivity from this private subnet CIDR block") | ||
ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN) | ||
}) | ||
|
||
AfterEach(func() { | ||
By("Reverting aws-node env variables to default values") | ||
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ | ||
"AWS_VPC_K8S_CNI_EXTERNALSNAT": "false", | ||
"AWS_VPC_K8S_CNI_RANDOMIZESNAT": "prng", | ||
}) | ||
k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{ | ||
"AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": {}, | ||
}) | ||
}) | ||
}) | ||
|
||
func ValidateExternalDomainConnectivity(url string) { | ||
testerArgs := []string{ | ||
"-testExternalDomainConnectivity=true", | ||
fmt.Sprintf("-url=%s", url), | ||
} | ||
|
||
testContainer := manifest.NewTestHelperContainer(). | ||
Command([]string{"./snat-utils"}). | ||
Args(testerArgs). | ||
Build() | ||
|
||
testPodManifest := manifest.NewDefaultPodBuilder(). | ||
Container(testContainer). | ||
NodeName(primaryNodeInPrivateSubnet.Name). | ||
Name("snat-test-pod"). | ||
Build() | ||
|
||
By("Deploying a test pod to check External domain access") | ||
testPod, err := f.K8sResourceManagers.PodManager(). | ||
CreateAndWaitTillPodCompleted(testPodManifest) | ||
Expect(err).NotTo(HaveOccurred()) | ||
|
||
logs, errLogs := f.K8sResourceManagers.PodManager(). | ||
PodLogs(testPod.Namespace, testPod.Name) | ||
Expect(errLogs).ToNot(HaveOccurred()) | ||
|
||
fmt.Fprintln(GinkgoWriter, logs) | ||
|
||
By("deleting the test pod") | ||
err = f.K8sResourceManagers.PodManager(). | ||
DeleteAndWaitTillPodDeleted(testPod) | ||
Expect(err).ToNot(HaveOccurred()) | ||
} | ||
|
||
func ValidateIPTableRules(randomizedSNATValue string, numOfCidrs int) { | ||
testerArgs := []string{ | ||
"-testIPTableRules=true", | ||
fmt.Sprintf("-randomizedSNATValue=%s", randomizedSNATValue), | ||
fmt.Sprintf("-numOfCidrs=%d", numOfCidrs), | ||
} | ||
|
||
hostNetworkContainer := manifest.NewTestHelperContainer(). | ||
Command([]string{"./snat-utils"}). | ||
CapabilitiesForSecurityContext([]corev1.Capability{ | ||
"NET_ADMIN", | ||
}, nil). | ||
Args(testerArgs). | ||
Build() | ||
|
||
hostNetworkPodManifest := manifest.NewDefaultPodBuilder(). | ||
Container(hostNetworkContainer). | ||
NodeName(primaryNodeInPublicSubnet.Name). | ||
Name("host-network"). | ||
HostNetwork(true). | ||
Build() | ||
|
||
By("creating pod to check iptable SNAT rules on the host") | ||
hostNetworkPod, err := f.K8sResourceManagers.PodManager(). | ||
CreateAndWaitTillPodCompleted(hostNetworkPodManifest) | ||
Expect(err).NotTo(HaveOccurred()) | ||
|
||
logs, errLogs := f.K8sResourceManagers.PodManager(). | ||
PodLogs(hostNetworkPod.Namespace, hostNetworkPod.Name) | ||
Expect(errLogs).ToNot(HaveOccurred()) | ||
|
||
fmt.Fprintln(GinkgoWriter, logs) | ||
|
||
By("deleting the host networking setup pod") | ||
err = f.K8sResourceManagers.PodManager(). | ||
DeleteAndWaitTillPodDeleted(hostNetworkPod) | ||
Expect(err).ToNot(HaveOccurred()) | ||
} |
Oops, something went wrong.