Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
resolved conflicts
Browse files Browse the repository at this point in the history
Refactored vpc_cni_logfile_test
Chinmay Gadgil committed Jul 15, 2021
1 parent 0af437a commit 44a6975
Showing 16 changed files with 803 additions and 140 deletions.
111 changes: 75 additions & 36 deletions go.sum

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions test/agent/Dockerfile
Original file line number Diff line number Diff line change
@@ -23,11 +23,15 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
-a -o networking cmd/networking/main.go

RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
-a -o snat-utils cmd/snat-utils/main.go

RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
-a -o metric-server cmd/metric-server/main.go

FROM public.ecr.aws/amazonlinux/amazonlinux:2
RUN yum update -y && \
yum install iptables -y && \
yum clean all

WORKDIR /
Binary file added test/agent/aws-node-config
Binary file not shown.
104 changes: 104 additions & 0 deletions test/agent/cmd/snat-utils/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package main

import (
"flag"
"fmt"
"log"
"net/http"
"strings"
"time"

"github.com/coreos/go-iptables/iptables"
)

func main() {
var testIPTableRules bool
var testExternalDomainConnectivity bool
var randomizedSNATValue string
var numOfCidrs int
var url string

flag.BoolVar(&testIPTableRules, "testIPTableRules", false, "bool flag when set to true tests validate if IPTable has required rules")
flag.StringVar(&randomizedSNATValue, "randomizedSNATValue", "prng", "value for AWS_VPC_K8S_CNI_RANDOMIZESNAT")
flag.IntVar(&numOfCidrs, "numOfCidrs", 1, "Number of CIDR blocks in customer VPC")
flag.BoolVar(&testExternalDomainConnectivity, "testExternalDomainConnectivity", false, "bool flag when set to true tests if the pod has internet access")
flag.StringVar(&url, "url", "https://aws.amazon.com/", "url to check for connectivity")

flag.Parse()

if testIPTableRules {
err := validateIPTableRules(randomizedSNATValue, numOfCidrs)
if err != nil {
log.Fatal(err)
}
log.Printf("Randomized SNAT test passed for AWS_VPC_K8S_CNI_RANDOMIZESNAT: %s\n", randomizedSNATValue)
}

if testExternalDomainConnectivity {
err := validateExternalDomainConnectivity(url)
if err != nil {
log.Fatal(err)
}
log.Println("External Domain Connectivity test passed")
}
}

func validateExternalDomainConnectivity(url string) error {
timeout := time.Duration(120 * time.Second)
client := http.Client{
Timeout: timeout,
}
resp, err := client.Get(url)
if err != nil {
return err
}

if resp.StatusCode != 200 {
return fmt.Errorf("%s returned response code: %d", url, resp.StatusCode)
}
return nil
}

func validateIPTableRules(randomizedSNATValue string, numOfCidrs int) error {
// Check IPTable rules corresponding to AWS_VPC_K8S_CNI_RANDOMIZESNAT
expectedString := "random-fully"
iptables, err := iptables.New()
if err != nil {
return err
}

if !iptables.HasRandomFully() || randomizedSNATValue == "hashrandom" {
expectedString = "random"
}

containsExpectedString := false
rule := ""

for i := 0; i <= numOfCidrs; i++ {
curr := fmt.Sprintf("AWS-SNAT-CHAIN-%d", i)
fmt.Printf("Checking: %s\n", curr)
chains, err := iptables.List("nat", curr)
if err != nil {
return err
}

for _, chain := range chains {
if strings.Contains(chain, expectedString) {
rule = chain
containsExpectedString = true
break
}
}

if containsExpectedString {
break
}
}

if randomizedSNATValue == "none" && containsExpectedString {
return fmt.Errorf("failed: found unexpected %s for SNAT rule: %s", expectedString, rule)
} else if randomizedSNATValue != "none" && !containsExpectedString {
return fmt.Errorf("failed: did not find expected %s for any of the SNAT rules", expectedString)
}
return nil
}
Binary file added test/agent/cni
Binary file not shown.
1 change: 1 addition & 0 deletions test/agent/go.mod
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ module github.com/aws/amazon-vpc-cni-k8s/test/agent
go 1.14

require (
github.com/coreos/go-iptables v0.6.0
github.com/vishvananda/netlink v1.1.0
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
)
2 changes: 2 additions & 0 deletions test/agent/go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
153 changes: 153 additions & 0 deletions test/e2e/snat/snat_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package snat

import (
"fmt"
"testing"

"github.com/aws/amazon-vpc-cni-k8s/test/framework"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils"
testUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
"github.com/aws/aws-sdk-go/aws"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
)

var (
f *framework.Framework
props utils.NodeGroupProperties
primaryNodeInPublicSubnet, primaryNodeInPrivateSubnet v1.Node
privateSubnetId string
input string
)

// Change this if you want to use your own Key Pair
const DEFAULT_KEY_PAIR = "test-key-pair"

func TestSnat(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Snat Suite")
}

var _ = BeforeSuite(func() {
f = framework.New(framework.GlobalOptions)

By("creating test namespace")
f.K8sResourceManagers.NamespaceManager().
CreateNamespace(testUtils.DefaultTestNamespace)

By("Getting existing nodes in the cluster")
nodes, err := f.K8sResourceManagers.NodeManager().GetAllNodes()
Expect(err).ToNot(HaveOccurred())

By("verifying more than 1 nodes are present for the test")
Expect(len(nodes.Items)).Should(BeNumerically(">", 1))

// Set the primary node for testing
primaryNodeInPublicSubnet = nodes.Items[0]

By("Getting Public and Private subnets")
vpcConfig, err := utils.GetClusterVPCConfig(f)
Expect(err).ToNot(HaveOccurred())

Expect(len(vpcConfig.PublicSubnetList)).To(BeNumerically(">", 0))
Expect(len(vpcConfig.PrivateSubnetList)).To(BeNumerically(">", 0))

msg := fmt.Sprintf("Creating a keyPair with name: %s if it doesn't exist", DEFAULT_KEY_PAIR)
By(msg)
keyPairOutput, _ := f.CloudServices.EC2().DescribeKey(DEFAULT_KEY_PAIR)

exists := false
if keyPairOutput != nil {
for _, keyPair := range keyPairOutput.KeyPairs {
if *keyPair.KeyName == DEFAULT_KEY_PAIR {
exists = true
break
}
}
}

if exists {
fmt.Println("KeyPair already exists")
} else {
fmt.Println("KeyPair doesn't exist, will be created")
_, err := f.CloudServices.EC2().CreateKey(DEFAULT_KEY_PAIR)
Expect(err).NotTo(HaveOccurred())
}

privateSubnetId = vpcConfig.PrivateSubnetList[0]

By("Getting Cluster Security Group Id")
out, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName)
Expect(err).NotTo(HaveOccurred())

clusterSecurityGroupId := out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId

msg = fmt.Sprintf("Deploying a self managed nodegroup of size 1 in private subnet %s", privateSubnetId)
By(msg)
props = utils.NodeGroupProperties{
NgLabelKey: "test-label-key",
NgLabelVal: "test-label-val",
AsgSize: 1,
NodeGroupName: "snat-test-ng",
Subnet: []string{
privateSubnetId,
},
InstanceType: "m5.large",
KeyPairName: DEFAULT_KEY_PAIR,
}

err = utils.CreateAndWaitTillSelfManagedNGReady(f, props)
Expect(err).NotTo(HaveOccurred())

nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(props.NgLabelKey,
props.NgLabelVal)
Expect(err).ToNot(HaveOccurred())
Expect(len(nodeList.Items)).Should(BeNumerically(">", 0))

// Get ref to the only node from newly created nodegroup
primaryNodeInPrivateSubnet = nodeList.Items[0]

By("Fetching existing Security Groups from the newly created node group instance")
instanceOutput, err := f.CloudServices.EC2().DescribeInstancesWithFilters(map[*string][]*string{
aws.String("private-dns-name"): {
aws.String(primaryNodeInPrivateSubnet.Name),
},
})

Expect(err).NotTo(HaveOccurred())
Expect(len(instanceOutput.Reservations)).To(BeNumerically(">", 0))
Expect(len(instanceOutput.Reservations[0].Instances)).To(BeNumerically(">", 0))

instance := instanceOutput.Reservations[0].Instances[0]

existingSecurityGroups := instance.SecurityGroups
networkInterfaceId := f.CloudServices.EC2().GetPrimaryNetworkInterfaceId(instance.NetworkInterfaces, instance.PrivateIpAddress)
Expect(networkInterfaceId).NotTo(Equal(BeNil()))

securityGroupIds := make([]*string, 0, len(existingSecurityGroups)+1)
for _, sg := range existingSecurityGroups {
securityGroupIds = append(securityGroupIds, sg.GroupId)
}
securityGroupIds = append(securityGroupIds, clusterSecurityGroupId)
By("Adding ClusterSecurityGroup to the new nodegroup Instance")
_, err = f.CloudServices.EC2().ModifyNetworkInterfaceSecurityGroups(securityGroupIds, networkInterfaceId)
Expect(err).NotTo(HaveOccurred())
})

var _ = AfterSuite(func() {
//using default key pair created by test
if DEFAULT_KEY_PAIR == "test-key-pair" {
By("Deleting key-pair")
err := f.CloudServices.EC2().DeleteKey(DEFAULT_KEY_PAIR)
Expect(err).NotTo(HaveOccurred())
}

By("Deleting test namespace")
f.K8sResourceManagers.NamespaceManager().
DeleteAndWaitTillNamespaceDeleted(testUtils.DefaultTestNamespace)

By("Deleting Managed Nodegroup")
err := utils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, props)
Expect(err).NotTo(HaveOccurred())
})
Loading

0 comments on commit 44a6975

Please sign in to comment.