Skip to content

Commit

Permalink
SNAT test changes
Browse files Browse the repository at this point in the history
WIP: Partal commit

minor change

SNAT test changes

WIP: Partal commit

minor change

Added SNAT tests for validating
AWS_VPC_K8S_CNI_EXTERNALSNAT
AWS_VPC_K8S_CNI_RANDOMIZESNAT
AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS

ref: https://quip-amazon.com/4oG5AcaS2VaP/CNI-Automation-Plan#s:CTQ9CAuByTC;CTQ9CAd8OVS

Moved env_vars_test file under cni
code cleanup

Added snat-utils for the agent

code cleanup
  • Loading branch information
Chinmay Gadgil committed Jun 17, 2021
1 parent 93edc95 commit 33ccdf0
Show file tree
Hide file tree
Showing 16 changed files with 701 additions and 6 deletions.
4 changes: 4 additions & 0 deletions test/agent/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,15 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
-a -o networking cmd/networking/main.go

RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
-a -o snat-utils cmd/snat-utils/main.go

RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \
-a -o metric-server cmd/metric-server/main.go

FROM public.ecr.aws/amazonlinux/amazonlinux:2
RUN yum update -y && \
yum install iptables -y && \
yum clean all

WORKDIR /
Expand Down
Binary file added test/agent/aws-node-config
Binary file not shown.
93 changes: 93 additions & 0 deletions test/agent/cmd/snat-utils/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package main

import (
"flag"
"fmt"
"log"
"net/http"
"strings"
"time"

"github.com/coreos/go-iptables/iptables"
)

func main() {
var testIPTableRules bool
var testExternalDomainConnectivity bool
var randomizedSNATValue string
var url string

flag.BoolVar(&testIPTableRules, "testIPTableRules", false, "bool flag when set to true tests validate if IPTable has required rules")
flag.StringVar(&randomizedSNATValue, "randomizedSNATValue", "prng", "value for AWS_VPC_K8S_CNI_RANDOMIZESNAT")
flag.BoolVar(&testExternalDomainConnectivity, "testExternalDomainConnectivity", false, "bool flag when set to true tests if the pod has internet access")
flag.StringVar(&url, "url", "https://aws.amazon.com/", "url to check for connectivity")

flag.Parse()

if testIPTableRules {
err := validateIPTableRules(randomizedSNATValue)
if err != nil {
log.Fatal(err)
}
log.Printf("Randomized SNAT test passed for AWS_VPC_K8S_CNI_RANDOMIZESNAT: %s\n", randomizedSNATValue)
}

if testExternalDomainConnectivity {
err := validateExternalDomainConnectivity(url)
if err != nil {
log.Fatal(err)
}
log.Println("External Domain Connectivity test passed")
}
}

func validateExternalDomainConnectivity(url string) error {
timeout := time.Duration(120 * time.Second)
client := http.Client{
Timeout: timeout,
}
resp, err := client.Get(url)
if err != nil {
return err
}

if resp.StatusCode != 200 {
return fmt.Errorf("%s returned response code: %d", url, resp.StatusCode)
}
return nil
}

func validateIPTableRules(randomizedSNATValue string) error {
// Check IPTable rules corresponding to AWS_VPC_K8S_CNI_RANDOMIZESNAT
expectedString := "random-fully"
iptables, err := iptables.New()
if err != nil {
return err
}

if !iptables.HasRandomFully() || randomizedSNATValue == "hashrandom" {
expectedString = "random"
}

chains, err := iptables.List("nat", "AWS-SNAT-CHAIN-1")
if err != nil {
return err
}

containsExpectedString := false
rule := ""
for _, chain := range chains {
if strings.Contains(chain, expectedString) {
rule = chain
containsExpectedString = true
break
}
}

if randomizedSNATValue == "none" && containsExpectedString {
return fmt.Errorf("failed: found unexpected %s for SNAT rule: %s", expectedString, rule)
} else if randomizedSNATValue != "none" && !containsExpectedString {
return fmt.Errorf("failed: did not find expected %s for any of the SNAT rules", expectedString)
}
return nil
}
Binary file added test/agent/cni
Binary file not shown.
1 change: 1 addition & 0 deletions test/agent/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/aws/amazon-vpc-cni-k8s/test/agent
go 1.14

require (
github.com/coreos/go-iptables v0.6.0
github.com/vishvananda/netlink v1.1.0
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
)
2 changes: 2 additions & 0 deletions test/agent/go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
Expand Down
2 changes: 2 additions & 0 deletions test/framework/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type Options struct {
NgNameLabelKey string
NgNameLabelVal string
EKSEndpoint string
KeyPair string
}

func (options *Options) BindFlags() {
Expand All @@ -44,6 +45,7 @@ func (options *Options) BindFlags() {
flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name")
flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name")
flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint")
flag.StringVar(&options.KeyPair, "key-pair", "test-key-pair", "optional, user specified key-pair to use if creating new nodes")
}

func (options *Options) Validate() error {
Expand Down
55 changes: 55 additions & 0 deletions test/framework/resources/aws/services/ec2.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
type EC2 interface {
DescribeInstanceType(instanceType string) ([]*ec2.InstanceTypeInfo, error)
DescribeInstance(instanceID string) (*ec2.Instance, error)
DescribeInstancesWithFilters(filterMap map[*string][]*string) (*ec2.DescribeInstancesOutput, error)
DescribeVPC(vpcID string) (*ec2.DescribeVpcsOutput, error)
DescribeNetworkInterface(interfaceIDs []string) (*ec2.DescribeNetworkInterfacesOutput, error)
AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
Expand All @@ -38,17 +39,30 @@ type EC2 interface {
CreateSubnet(cidrBlock string, vpcID string, az string) (*ec2.CreateSubnetOutput, error)
DeleteSubnet(subnetID string) error
DescribeRouteTables(subnetID string) (*ec2.DescribeRouteTablesOutput, error)
DescribeRouteTablesWithVPCID(vpcID string) (*ec2.DescribeRouteTablesOutput, error)
CreateSecurityGroup(groupName string, description string, vpcID string) (*ec2.CreateSecurityGroupOutput, error)
DeleteSecurityGroup(groupID string) error
AssociateRouteTableToSubnet(routeTableId string, subnetID string) error
CreateKey(keyName string) (*ec2.CreateKeyPairOutput, error)
DeleteKey(keyName string) error
DescribeKey(keyName string) (*ec2.DescribeKeyPairsOutput, error)
ModifyNetworkInterfaceSecurityGroups(securityGroupIds []*string, networkInterfaceId *string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error)
GetPrimaryNetworkInterfaceId([]*ec2.InstanceNetworkInterface) *string
}

type defaultEC2 struct {
ec2iface.EC2API
}

func (d *defaultEC2) GetPrimaryNetworkInterfaceId(networkInterfaces []*ec2.InstanceNetworkInterface) *string {
for _, ni := range networkInterfaces {
if aws.Int64Value(ni.Attachment.DeviceIndex) == 0 {
return ni.NetworkInterfaceId
}
}
return nil
}

func (d *defaultEC2) DescribeInstanceType(instanceType string) ([]*ec2.InstanceTypeInfo, error) {
describeInstanceTypeIp := &ec2.DescribeInstanceTypesInput{
InstanceTypes: aws.StringSlice([]string{instanceType}),
Expand All @@ -63,6 +77,13 @@ func (d *defaultEC2) DescribeInstanceType(instanceType string) ([]*ec2.InstanceT
return describeInstanceOp.InstanceTypes, nil
}

func (d *defaultEC2) ModifyNetworkInterfaceSecurityGroups(securityGroupIds []*string, networkInterfaceId *string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) {
return d.EC2API.ModifyNetworkInterfaceAttribute(&ec2.ModifyNetworkInterfaceAttributeInput{
NetworkInterfaceId: networkInterfaceId,
Groups: securityGroupIds,
})
}

func (d *defaultEC2) DescribeInstance(instanceID string) (*ec2.Instance, error) {
describeInstanceInput := &ec2.DescribeInstancesInput{
InstanceIds: aws.StringSlice([]string{instanceID}),
Expand All @@ -78,6 +99,19 @@ func (d *defaultEC2) DescribeInstance(instanceID string) (*ec2.Instance, error)
return describeInstanceOutput.Reservations[0].Instances[0], nil
}

func (d *defaultEC2) DescribeInstancesWithFilters(filterMap map[*string][]*string) (*ec2.DescribeInstancesOutput, error) {
filters := []*ec2.Filter{}
for k, v := range filterMap {
filters = append(filters, &ec2.Filter{
Name: k,
Values: v,
})
}
return d.EC2API.DescribeInstances(&ec2.DescribeInstancesInput{
Filters: filters,
})
}

func (d *defaultEC2) AuthorizeSecurityGroupIngress(groupID string, protocol string,
fromPort int, toPort int, cidrIP string) error {
ipPermissions := &ec2.IpPermission{
Expand Down Expand Up @@ -197,6 +231,18 @@ func (d *defaultEC2) DescribeSubnet(subnetID string) (*ec2.DescribeSubnetsOutput
return d.EC2API.DescribeSubnets(describeSubnetInput)
}

func (d *defaultEC2) DescribeRouteTablesWithVPCID(vpcID string) (*ec2.DescribeRouteTablesOutput, error) {
describeRouteTableInput := &ec2.DescribeRouteTablesInput{
Filters: []*ec2.Filter{
{
Name: aws.String("vpc-id"),
Values: aws.StringSlice([]string{vpcID}),
},
},
}
return d.EC2API.DescribeRouteTables(describeRouteTableInput)
}

func (d *defaultEC2) DeleteSubnet(subnetID string) error {
deleteSubnetInput := &ec2.DeleteSubnetInput{
SubnetId: aws.String(subnetID),
Expand Down Expand Up @@ -260,6 +306,15 @@ func (d *defaultEC2) DeleteKey(keyName string) error {
return err
}

func (d *defaultEC2) DescribeKey(keyName string) (*ec2.DescribeKeyPairsOutput, error) {
keyPairInput := &ec2.DescribeKeyPairsInput{
KeyNames: []*string{
&keyName,
},
}
return d.EC2API.DescribeKeyPairs(keyPairInput)
}

func (d *defaultEC2) TerminateInstance(instanceIDs []string) error {
terminateInstanceInput := &ec2.TerminateInstancesInput{
DryRun: nil,
Expand Down
12 changes: 10 additions & 2 deletions test/framework/resources/aws/utils/nodegroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ type ClusterVPCConfig struct {
PublicSubnetList []string
AvailZones []string
PublicRouteTableID string
PrivateSubnetList []string
}

type AWSAuthMapRole struct {
Expand Down Expand Up @@ -207,8 +208,9 @@ func DeleteAndWaitTillSelfManagedNGStackDeleted(f *framework.Framework, properti

func GetClusterVPCConfig(f *framework.Framework) (*ClusterVPCConfig, error) {
clusterConfig := &ClusterVPCConfig{
PublicSubnetList: []string{},
AvailZones: []string{},
PublicSubnetList: []string{},
AvailZones: []string{},
PrivateSubnetList: []string{},
}

describeClusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName)
Expand All @@ -221,12 +223,18 @@ func GetClusterVPCConfig(f *framework.Framework) (*ClusterVPCConfig, error) {
if err != nil {
return nil, fmt.Errorf("failed to describe subnet %s: %v", *subnet, err)
}

isPublic := false
for _, route := range describeRouteOutput.RouteTables[0].Routes {
if route.GatewayId != nil && strings.Contains(*route.GatewayId, "igw-") {
isPublic = true
clusterConfig.PublicSubnetList = append(clusterConfig.PublicSubnetList, *subnet)
clusterConfig.PublicRouteTableID = *describeRouteOutput.RouteTables[0].RouteTableId
}
}
if !isPublic {
clusterConfig.PrivateSubnetList = append(clusterConfig.PrivateSubnetList, *subnet)
}
}

uniqueAZ := map[string]bool{}
Expand Down
12 changes: 12 additions & 0 deletions test/framework/resources/k8s/manifest/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type Container struct {
args []string
probe *v1.Probe
ports []v1.ContainerPort
securityContext *v1.SecurityContext
}

func NewBusyBoxContainerBuilder() *Container {
Expand Down Expand Up @@ -64,6 +65,16 @@ func NewNetCatAlpineContainer() *Container {
}
}

func (w *Container) CapabilitiesForSecurityContext(add []v1.Capability, drop []v1.Capability) *Container {
w.securityContext = &v1.SecurityContext{
Capabilities: &v1.Capabilities{
Add: add,
Drop: drop,
},
}
return w
}

func (w *Container) Name(name string) *Container {
w.name = name
return w
Expand Down Expand Up @@ -108,5 +119,6 @@ func (w *Container) Build() v1.Container {
ImagePullPolicy: w.imagePullPolicy,
LivenessProbe: w.probe,
Ports: w.ports,
SecurityContext: w.securityContext,
}
}
Loading

0 comments on commit 33ccdf0

Please sign in to comment.