diff --git a/test/agent/go.sum b/test/agent/go.sum index 71f61a9b28..ad48fb3570 100644 --- a/test/agent/go.sum +++ b/test/agent/go.sum @@ -1,3 +1,5 @@ +github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= diff --git a/test/e2e/README.md b/test/e2e/README.md index 197a15a010..73f8e52835 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -7,6 +7,8 @@ The package contains e2e tests suites for `amazon-vpc-cni-k8s` . - No existing node group should be present the test creates new self managed node group with the reduced MAX_POD value. - Security Group For Pods Test - EKS Cluster should be v1.16+. This tests creates an additional Trunk ENI on all Nitro based instance present in the cluster. This could interfere with running integration test that test WARM_ENI_TARGET. For this reasons the test should either be run without any node group present in the cluster or at the very end. +- Snat Test + - EKS Cluster should have atleast 1 private subnet and atleast 1 public subnet. These tests modifies the SNAT related variabls in aws-node, validates the IP table SNAT rules and checks for Internet Connectivity. ####Testing Set the environment variables that will be passed to Ginkgo script. If you want to directly pass the arguments you can skip to next step. diff --git a/test/e2e/snat/snat_suite_test.go b/test/e2e/snat/snat_suite_test.go new file mode 100644 index 0000000000..097749f168 --- /dev/null +++ b/test/e2e/snat/snat_suite_test.go @@ -0,0 +1,165 @@ +package snat + +import ( + "fmt" + "net/url" + "path" + "strings" + "testing" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils" + testUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" +) + +var ( + f *framework.Framework + props utils.NodeGroupProperties + primaryNodeInPublicSubnet, primaryNodeInPrivateSubnet v1.Node + privateSubnetId string + input string +) + +// Change this if you want to use your own Key Pair +const DEFAULT_KEY_PAIR = "test-key-pair" + +func TestSnat(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Snat Suite") +} + +var _ = BeforeSuite(func() { + f = framework.New(framework.GlobalOptions) + + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(testUtils.DefaultTestNamespace) + + By("Getting existing nodes in the cluster") + nodes, err := f.K8sResourceManagers.NodeManager().GetAllNodes() + Expect(err).ToNot(HaveOccurred()) + + By("verifying more than 1 nodes are present for the test") + Expect(len(nodes.Items)).Should(BeNumerically(">", 1)) + + // Set the primary node for testing + primaryNodeInPublicSubnet = nodes.Items[0] + + By("Getting Public and Private subnets") + vpcConfig, err := utils.GetClusterVPCConfig(f) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(vpcConfig.PublicSubnetList)).To(BeNumerically(">", 0)) + Expect(len(vpcConfig.PrivateSubnetList)).To(BeNumerically(">", 0)) + + msg := fmt.Sprintf("Creating a keyPair with name: %s if it doesn't exist", DEFAULT_KEY_PAIR) + By(msg) + keyPairOutput, _ := f.CloudServices.EC2().DescribeKey(DEFAULT_KEY_PAIR) + + exists := false + if keyPairOutput != nil { + for _, keyPair := range keyPairOutput.KeyPairs { + if *keyPair.KeyName == DEFAULT_KEY_PAIR { + exists = true + break + } + } + } + + if exists { + fmt.Fprintln(GinkgoWriter, "KeyPair already exists") + } else { + fmt.Fprintln(GinkgoWriter, "KeyPair doesn't exist, will be created") + _, err := f.CloudServices.EC2().CreateKey(DEFAULT_KEY_PAIR) + Expect(err).NotTo(HaveOccurred()) + } + + privateSubnetId = vpcConfig.PrivateSubnetList[0] + + By("Getting Cluster Security Group Id") + out, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) + Expect(err).NotTo(HaveOccurred()) + + clusterSecurityGroupId := out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId + + msg = fmt.Sprintf("Deploying a self managed nodegroup of size 1 in private subnet %s", privateSubnetId) + By(msg) + props = utils.NodeGroupProperties{ + NgLabelKey: "test-label-key", + NgLabelVal: "test-label-val", + AsgSize: 1, + NodeGroupName: "snat-test-ng", + Subnet: []string{ + privateSubnetId, + }, + InstanceType: "m5.large", + KeyPairName: DEFAULT_KEY_PAIR, + } + + err = utils.CreateAndWaitTillSelfManagedNGReady(f, props) + Expect(err).NotTo(HaveOccurred()) + + nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(props.NgLabelKey, + props.NgLabelVal) + Expect(err).ToNot(HaveOccurred()) + Expect(len(nodeList.Items)).Should(BeNumerically(">", 0)) + + // Get ref to the only node from newly created nodegroup + primaryNodeInPrivateSubnet = nodeList.Items[0] + providerID := primaryNodeInPrivateSubnet.Spec.ProviderID + Expect(len(providerID)).To(BeNumerically(">", 0)) + + awsUrl, err := url.Parse(providerID) + Expect(err).NotTo(HaveOccurred()) + + instanceID := path.Base(awsUrl.Path) + Expect(len(instanceID)).To(BeNumerically(">", 0)) + + By("Fetching existing Security Groups from the newly created node group instance") + + instance, err := f.CloudServices.EC2().DescribeInstance(instanceID) + Expect(err).NotTo(HaveOccurred()) + + existingSecurityGroups := instance.SecurityGroups + networkInterfaceId := getPrimaryNetworkInterfaceId(instance.NetworkInterfaces, instance.PrivateIpAddress) + Expect(networkInterfaceId).NotTo(Equal(BeNil())) + + securityGroupIds := make([]*string, 0, len(existingSecurityGroups)+1) + for _, sg := range existingSecurityGroups { + securityGroupIds = append(securityGroupIds, sg.GroupId) + } + securityGroupIds = append(securityGroupIds, clusterSecurityGroupId) + By("Adding ClusterSecurityGroup to the new nodegroup Instance") + _, err = f.CloudServices.EC2().ModifyNetworkInterfaceSecurityGroups(securityGroupIds, networkInterfaceId) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + //using default key pair created by test + if DEFAULT_KEY_PAIR == "test-key-pair" { + By("Deleting key-pair") + err := f.CloudServices.EC2().DeleteKey(DEFAULT_KEY_PAIR) + Expect(err).NotTo(HaveOccurred()) + } + + By("Deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(testUtils.DefaultTestNamespace) + + By("Deleting Managed Nodegroup") + err := utils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, props) + Expect(err).NotTo(HaveOccurred()) +}) + +func getPrimaryNetworkInterfaceId(networkInterfaces []*ec2.InstanceNetworkInterface, instanceIPAddr *string) *string { + for _, ni := range networkInterfaces { + if strings.Compare(*ni.PrivateIpAddress, *instanceIPAddr) == 0 { + return ni.NetworkInterfaceId + } + } + return nil +} diff --git a/test/e2e/snat/snat_test.go b/test/e2e/snat/snat_test.go new file mode 100644 index 0000000000..2e47aee7e6 --- /dev/null +++ b/test/e2e/snat/snat_test.go @@ -0,0 +1,170 @@ +package snat + +import ( + "fmt" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/aws-sdk-go/aws" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +const ( + TEST_POD_LABEL_KEY = "test-pod-label-key" + TEST_POD_LABEL_VALUE = "test-pod-label-val" + EXTERNAL_DOMAIN = "https://aws.amazon.com/" +) + +var _ = Describe("SNAT tests", func() { + Context("ExternalSnat=false", func() { + BeforeEach(func() { + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "AWS_VPC_K8S_CNI_EXTERNALSNAT": "false", + }) + }) + + It("Pod in private subnet should have Internet access with External SNAT disabled", func() { + By("Checking External Domain Connectivity") + ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN) + }) + }) + + Context("ExternSnat=true", func() { + BeforeEach(func() { + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "AWS_VPC_K8S_CNI_EXTERNALSNAT": "true", + }) + }) + + It("Pod in private subnet should have Internet access with External SNAT enabled", func() { + By("Checking External Domain Connectivity") + ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN) + }) + }) + + Context("Validate AWS_VPC_K8S_CNI_RANDOMIZESNAT", func() { + It("Verify SNAT IP table rule by changing AWS_VPC_K8S_CNI_RANDOMIZESNAT", func() { + vpcOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID) + Expect(err).NotTo(HaveOccurred()) + Expect(len(vpcOutput.Vpcs)).To(BeNumerically(">", 0)) + + numOfCidrs := len(vpcOutput.Vpcs[0].CidrBlockAssociationSet) + + By("Check whether SNAT IP table has random-fully with AWS_VPC_K8S_CNI_RANDOMIZESNAT set to default value of prng") + ValidateIPTableRules("prng", numOfCidrs) + + By("Setting AWS_VPC_K8S_CNI_RANDOMIZESNAT to none") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "AWS_VPC_K8S_CNI_RANDOMIZESNAT": "none", + }) + + By("Check where SNAT IP table rule is updated and it doesn't contain random port allocation") + ValidateIPTableRules("none", numOfCidrs) + }) + }) + + Context("Validate AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS", func() { + It("Verify External Domain Connectivity by modifying AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS", func() { + By("Getting CIDR for primary node's private subnet") + out, err := f.CloudServices.EC2().DescribeSubnet(privateSubnetId) + Expect(err).NotTo(HaveOccurred()) + Expect(len(out.Subnets)).To(BeNumerically(">", 0)) + + cidrBlock := out.Subnets[0].CidrBlock + By("Updating AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS with private subnet CIDR") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": aws.StringValue(cidrBlock), + }) + + By("Check External domain connectivity from this private subnet CIDR block") + ValidateExternalDomainConnectivity(EXTERNAL_DOMAIN) + }) + }) + + AfterEach(func() { + By("Reverting aws-node env variables to default values") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "AWS_VPC_K8S_CNI_EXTERNALSNAT": "false", + "AWS_VPC_K8S_CNI_RANDOMIZESNAT": "prng", + }) + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{ + "AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS": {}, + }) + }) +}) + +func ValidateExternalDomainConnectivity(url string) { + testerArgs := []string{ + "-testExternalDomainConnectivity=true", + fmt.Sprintf("-url=%s", url), + } + + testContainer := manifest.NewTestHelperContainer(). + Command([]string{"./snat-utils"}). + Args(testerArgs). + Build() + + testPodManifest := manifest.NewDefaultPodBuilder(). + Container(testContainer). + NodeName(primaryNodeInPrivateSubnet.Name). + Name("snat-test-pod"). + Build() + + By("Deploying a test pod to check External domain access") + testPod, err := f.K8sResourceManagers.PodManager(). + CreateAndWaitTillPodCompleted(testPodManifest) + Expect(err).NotTo(HaveOccurred()) + + logs, errLogs := f.K8sResourceManagers.PodManager(). + PodLogs(testPod.Namespace, testPod.Name) + Expect(errLogs).ToNot(HaveOccurred()) + + fmt.Fprintln(GinkgoWriter, logs) + + By("deleting the test pod") + err = f.K8sResourceManagers.PodManager(). + DeleteAndWaitTillPodDeleted(testPod) + Expect(err).ToNot(HaveOccurred()) +} + +func ValidateIPTableRules(randomizedSNATValue string, numOfCidrs int) { + testerArgs := []string{ + "-testIPTableRules=true", + fmt.Sprintf("-randomizedSNATValue=%s", randomizedSNATValue), + fmt.Sprintf("-numOfCidrs=%d", numOfCidrs), + } + + hostNetworkContainer := manifest.NewTestHelperContainer(). + Command([]string{"./snat-utils"}). + CapabilitiesForSecurityContext([]corev1.Capability{ + "NET_ADMIN", + }, nil). + Args(testerArgs). + Build() + + hostNetworkPodManifest := manifest.NewDefaultPodBuilder(). + Container(hostNetworkContainer). + NodeName(primaryNodeInPublicSubnet.Name). + Name("host-network"). + HostNetwork(true). + Build() + + By("creating pod to check iptable SNAT rules on the host") + hostNetworkPod, err := f.K8sResourceManagers.PodManager(). + CreateAndWaitTillPodCompleted(hostNetworkPodManifest) + Expect(err).NotTo(HaveOccurred()) + + logs, errLogs := f.K8sResourceManagers.PodManager(). + PodLogs(hostNetworkPod.Namespace, hostNetworkPod.Name) + Expect(errLogs).ToNot(HaveOccurred()) + + fmt.Fprintln(GinkgoWriter, logs) + + By("deleting the host networking setup pod") + err = f.K8sResourceManagers.PodManager(). + DeleteAndWaitTillPodDeleted(hostNetworkPod) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/test/framework/resources/aws/services/ec2.go b/test/framework/resources/aws/services/ec2.go index 4271b8bb35..7eceaed67a 100644 --- a/test/framework/resources/aws/services/ec2.go +++ b/test/framework/resources/aws/services/ec2.go @@ -38,11 +38,14 @@ type EC2 interface { CreateSubnet(cidrBlock string, vpcID string, az string) (*ec2.CreateSubnetOutput, error) DeleteSubnet(subnetID string) error DescribeRouteTables(subnetID string) (*ec2.DescribeRouteTablesOutput, error) + DescribeRouteTablesWithVPCID(vpcID string) (*ec2.DescribeRouteTablesOutput, error) CreateSecurityGroup(groupName string, description string, vpcID string) (*ec2.CreateSecurityGroupOutput, error) DeleteSecurityGroup(groupID string) error AssociateRouteTableToSubnet(routeTableId string, subnetID string) error CreateKey(keyName string) (*ec2.CreateKeyPairOutput, error) DeleteKey(keyName string) error + DescribeKey(keyName string) (*ec2.DescribeKeyPairsOutput, error) + ModifyNetworkInterfaceSecurityGroups(securityGroupIds []*string, networkInterfaceId *string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) } type defaultEC2 struct { @@ -63,6 +66,13 @@ func (d *defaultEC2) DescribeInstanceType(instanceType string) ([]*ec2.InstanceT return describeInstanceOp.InstanceTypes, nil } +func (d *defaultEC2) ModifyNetworkInterfaceSecurityGroups(securityGroupIds []*string, networkInterfaceId *string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) { + return d.EC2API.ModifyNetworkInterfaceAttribute(&ec2.ModifyNetworkInterfaceAttributeInput{ + NetworkInterfaceId: networkInterfaceId, + Groups: securityGroupIds, + }) +} + func (d *defaultEC2) DescribeInstance(instanceID string) (*ec2.Instance, error) { describeInstanceInput := &ec2.DescribeInstancesInput{ InstanceIds: aws.StringSlice([]string{instanceID}), @@ -197,6 +207,18 @@ func (d *defaultEC2) DescribeSubnet(subnetID string) (*ec2.DescribeSubnetsOutput return d.EC2API.DescribeSubnets(describeSubnetInput) } +func (d *defaultEC2) DescribeRouteTablesWithVPCID(vpcID string) (*ec2.DescribeRouteTablesOutput, error) { + describeRouteTableInput := &ec2.DescribeRouteTablesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: aws.StringSlice([]string{vpcID}), + }, + }, + } + return d.EC2API.DescribeRouteTables(describeRouteTableInput) +} + func (d *defaultEC2) DeleteSubnet(subnetID string) error { deleteSubnetInput := &ec2.DeleteSubnetInput{ SubnetId: aws.String(subnetID), @@ -260,6 +282,15 @@ func (d *defaultEC2) DeleteKey(keyName string) error { return err } +func (d *defaultEC2) DescribeKey(keyName string) (*ec2.DescribeKeyPairsOutput, error) { + keyPairInput := &ec2.DescribeKeyPairsInput{ + KeyNames: []*string{ + &keyName, + }, + } + return d.EC2API.DescribeKeyPairs(keyPairInput) +} + func (d *defaultEC2) TerminateInstance(instanceIDs []string) error { terminateInstanceInput := &ec2.TerminateInstancesInput{ DryRun: nil, diff --git a/test/framework/resources/aws/utils/nodegroup.go b/test/framework/resources/aws/utils/nodegroup.go index 47c59930b2..b3a951dd42 100644 --- a/test/framework/resources/aws/utils/nodegroup.go +++ b/test/framework/resources/aws/utils/nodegroup.go @@ -50,6 +50,7 @@ type ClusterVPCConfig struct { PublicSubnetList []string AvailZones []string PublicRouteTableID string + PrivateSubnetList []string } type AWSAuthMapRole struct { @@ -207,8 +208,9 @@ func DeleteAndWaitTillSelfManagedNGStackDeleted(f *framework.Framework, properti func GetClusterVPCConfig(f *framework.Framework) (*ClusterVPCConfig, error) { clusterConfig := &ClusterVPCConfig{ - PublicSubnetList: []string{}, - AvailZones: []string{}, + PublicSubnetList: []string{}, + AvailZones: []string{}, + PrivateSubnetList: []string{}, } describeClusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) @@ -221,12 +223,18 @@ func GetClusterVPCConfig(f *framework.Framework) (*ClusterVPCConfig, error) { if err != nil { return nil, fmt.Errorf("failed to describe subnet %s: %v", *subnet, err) } + + isPublic := false for _, route := range describeRouteOutput.RouteTables[0].Routes { if route.GatewayId != nil && strings.Contains(*route.GatewayId, "igw-") { + isPublic = true clusterConfig.PublicSubnetList = append(clusterConfig.PublicSubnetList, *subnet) clusterConfig.PublicRouteTableID = *describeRouteOutput.RouteTables[0].RouteTableId } } + if !isPublic { + clusterConfig.PrivateSubnetList = append(clusterConfig.PrivateSubnetList, *subnet) + } } uniqueAZ := map[string]bool{} diff --git a/test/framework/resources/k8s/manifest/container.go b/test/framework/resources/k8s/manifest/container.go index 555803a4b6..c121354b70 100644 --- a/test/framework/resources/k8s/manifest/container.go +++ b/test/framework/resources/k8s/manifest/container.go @@ -27,6 +27,7 @@ type Container struct { args []string probe *v1.Probe ports []v1.ContainerPort + securityContext *v1.SecurityContext } func NewBusyBoxContainerBuilder() *Container { @@ -64,6 +65,16 @@ func NewNetCatAlpineContainer() *Container { } } +func (w *Container) CapabilitiesForSecurityContext(add []v1.Capability, drop []v1.Capability) *Container { + w.securityContext = &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: add, + Drop: drop, + }, + } + return w +} + func (w *Container) Name(name string) *Container { w.name = name return w @@ -108,5 +119,6 @@ func (w *Container) Build() v1.Container { ImagePullPolicy: w.imagePullPolicy, LivenessProbe: w.probe, Ports: w.ports, + SecurityContext: w.securityContext, } } diff --git a/test/framework/resources/k8s/utils/daemonset.go b/test/framework/resources/k8s/utils/daemonset.go index d5353df277..cf5f65c688 100644 --- a/test/framework/resources/k8s/utils/daemonset.go +++ b/test/framework/resources/k8s/utils/daemonset.go @@ -74,9 +74,9 @@ func updateDaemonsetEnvVarsAndWait(f *framework.Framework, dsName string, dsName // update multus daemonset if it exists // to avoid being stuck in recursive loop, we need below check if dsName != utils.MultusNodeName { - By("Restarting Multus daemonset if it exists") _, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(dsNamespace, utils.MultusNodeName) if err == nil { + By("Restarting Multus daemonset") td := time.Now() updateDaemonsetEnvVarsAndWait(f, utils.MultusNodeName, dsNamespace, utils.MultusContainerName, map[string]string{ "forceUpdatedAt": td.String(), diff --git a/test/integration-new/cni/vpc_cni_logfile_test.go b/test/integration-new/cni/vpc_cni_logfile_test.go new file mode 100644 index 0000000000..4836d0152d --- /dev/null +++ b/test/integration-new/cni/vpc_cni_logfile_test.go @@ -0,0 +1,94 @@ +package cni + +import ( + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsV1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" +) + +const ( + AWS_VPC_K8S_CNI_LOG_FILE = "AWS_VPC_K8S_CNI_LOG_FILE" + POD_VOL_LABEL_KEY = "MountVolume" + POD_VOL_LABEL_VAL = "true" + VOLUME_NAME = "ipamd-logs" + VOLUME_MOUNT_PATH = "/var/log/aws-routed-eni/" +) + +var _ = Describe("aws-node env test", func() { + var ( + deploymentSpecWithVol *appsV1.Deployment + ) + + var _ = JustBeforeEach(func() { + By("Deploying a host network deployment with Volume mount") + container := manifest.NewBusyBoxContainerBuilder().Build() + + volume := []v1.Volume{ + { + Name: VOLUME_NAME, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: VOLUME_MOUNT_PATH, + }, + }, + }, + } + + volumeMount := []v1.VolumeMount{ + { + Name: VOLUME_NAME, + MountPath: VOLUME_NAME, + }, + } + + deploymentSpecWithVol = manifest.NewDefaultDeploymentBuilder(). + Namespace("default"). + Name("host-network"). + Replicas(1). + HostNetwork(true). + Container(container). + PodLabel(POD_VOL_LABEL_KEY, POD_VOL_LABEL_VAL). + MountVolume(volume, volumeMount). + NodeName(primaryNode.Name). + Build() + + _, err := f.K8sResourceManagers. + DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deploymentSpecWithVol, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + }) + + It("aws-node environment variable AWS_VPC_K8S_CNI_LOG_FILE test", func() { + By("Checking for pod with volume mount") + pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(POD_VOL_LABEL_KEY, POD_VOL_LABEL_VAL) + Expect(err).NotTo(HaveOccurred()) + Expect(len(pods.Items)).Should(BeNumerically(">", 0)) + + podWithVol := pods.Items[0] + + newLogFile := "ipamd_test.log" + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + AWS_VPC_K8S_CNI_LOG_FILE: "/host/var/log/aws-routed-eni/" + newLogFile, + }) + + stdout, _, err := f.K8sResourceManagers.PodManager().PodExec("default", podWithVol.Name, []string{"tail", "-n", "5", "ipamd-logs/ipamd_test.log"}) + Expect(err).NotTo(HaveOccurred()) + Expect(stdout).NotTo(Equal("")) + + }) + + var _ = JustAfterEach(func() { + By("Restoring old value on daemonset") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + AWS_VPC_K8S_CNI_LOG_FILE: "/host/var/log/aws-routed-eni/ipamd.log", + }) + + By("Deleing deployment with Volume Mount") + err := f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deploymentSpecWithVol) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/test/integration-new/ipamd/eni_ip_leak_test.go b/test/integration-new/ipamd/eni_ip_leak_test.go index 4c82146a36..8bcece8efd 100644 --- a/test/integration-new/ipamd/eni_ip_leak_test.go +++ b/test/integration-new/ipamd/eni_ip_leak_test.go @@ -11,8 +11,6 @@ import ( ) const ( - NAMESPACE = "kube-system" - DAEMONSET = "aws-node" HOST_POD_LABEL_KEY = "network" HOST_POD_LABEL_VAL = "host" ) diff --git a/test/integration-new/ipamd/env_vars_test.go b/test/integration-new/ipamd/env_vars_test.go deleted file mode 100644 index 6760f24486..0000000000 --- a/test/integration-new/ipamd/env_vars_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package ipamd - -import ( - "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" - k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" - "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - appsV1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" -) - -var ( - ds *appsV1.DaemonSet -) - -const ( - AWS_VPC_K8S_CNI_LOG_FILE = "AWS_VPC_K8S_CNI_LOG_FILE" - POD_VOL_LABEL_KEY = "MountVolume" - POD_VOL_LABEL_VAL = "true" - VOLUME_NAME = "ipamd-logs" - VOLUME_MOUNT_PATH = "/var/log/aws-routed-eni/" - OLD_PATH = "/host/var/log/aws-routed-eni/ipamd.log" -) - -var _ = Describe("cni env test", func() { - Context("CNI Environment Variables", func() { - BeforeEach(func() { - By("creating test namespace") - f.K8sResourceManagers.NamespaceManager(). - CreateNamespace(utils.DefaultTestNamespace) - - }) - - It("Changing AWS_VPC_K8S_CNI_LOG_FILE", func() { - By("Deploying a host network deployment with Volume mount") - curlContainer := manifest.NewBusyBoxContainerBuilder().Image("curlimages/curl:7.76.1").Name("curler").Build() - - volume := []v1.Volume{ - { - Name: VOLUME_NAME, - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: VOLUME_MOUNT_PATH, - }, - }, - }, - } - - volumeMount := []v1.VolumeMount{ - { - Name: VOLUME_NAME, - MountPath: VOLUME_NAME, - }, - } - - deploymentSpecWithVol := manifest.NewDefaultDeploymentBuilder(). - Namespace(utils.DefaultTestNamespace). - Name("host-network"). - Replicas(1). - HostNetwork(true). - Container(curlContainer). - PodLabel(POD_VOL_LABEL_KEY, POD_VOL_LABEL_VAL). - MountVolume(volume, volumeMount). - NodeName(primaryNode.Name). - Build() - - _, err := f.K8sResourceManagers. - DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(deploymentSpecWithVol, utils.DefaultDeploymentReadyTimeout) - Expect(err).ToNot(HaveOccurred()) - - pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(POD_VOL_LABEL_KEY, POD_VOL_LABEL_VAL) - Expect(err).NotTo(HaveOccurred()) - Expect(len(pods.Items)).Should(BeNumerically(">", 0)) - - podWithVol := pods.Items[0] - - ds, err = f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(NAMESPACE, DAEMONSET) - Expect(err).NotTo(HaveOccurred()) - - newLogFile := "ipamd_test.log" - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, map[string]string{ - AWS_VPC_K8S_CNI_LOG_FILE: "/host/var/log/aws-routed-eni/" + newLogFile, - }) - - stdout, _, err := f.K8sResourceManagers.PodManager().PodExec(utils.DefaultTestNamespace, podWithVol.Name, []string{"tail", "-n", "5", "ipamd-logs/ipamd_test.log"}) - Expect(err).NotTo(HaveOccurred()) - Expect(stdout).NotTo(Equal("")) - }) - - AfterEach(func() { - By("deleting test namespace") - f.K8sResourceManagers.NamespaceManager(). - DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) - - By("Restoring old value on daemonset") - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, map[string]string{ - AWS_VPC_K8S_CNI_LOG_FILE: OLD_PATH, - }) - }) - }) -}) diff --git a/test/integration-new/ipamd/warm_target_test.go b/test/integration-new/ipamd/warm_target_test.go index 424b2cc2c2..5bceb67ceb 100644 --- a/test/integration-new/ipamd/warm_target_test.go +++ b/test/integration-new/ipamd/warm_target_test.go @@ -171,24 +171,3 @@ var _ = Describe("test warm target variables", func() { }) }) }) - -func Max(x, y int) int { - if x < y { - return y - } - return x -} - -// MinIgnoreZero returns smaller of two number, if any number is zero returns the other number -func MinIgnoreZero(x, y int) int { - if x == 0 { - return y - } - if y == 0 { - return x - } - if x < y { - return x - } - return y -}