Skip to content

Commit

Permalink
annotate pods with v6 addr from branch ENI when present (#309)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jdn5126
jdn5126 authored Oct 9, 2023
1 parent 8495249 commit 86d7623
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 34 deletions.
28 changes: 28 additions & 0 deletions mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 33 additions & 4 deletions pkg/aws/ec2/instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,18 @@ type ec2Instance struct {
// subnetId is the instance's subnet id
instanceSubnetID string
// instanceSubnetCidrBlock is the cidr block of the instance's subnet
instanceSubnetCidrBlock string
instanceSubnetCidrBlock string
instanceSubnetV6CidrBlock string
// currentSubnetID can either point to the Subnet ID of the instance or subnet ID from the ENIConfig
currentSubnetID string
// currentSubnetCIDRBlock can either point to the Subnet CIDR block for instance subnet or subnet from ENIConfig
currentSubnetCIDRBlock string
currentSubnetCIDRBlock string
currentSubnetV6CIDRBlock string
// currentInstanceSecurityGroups can either point to the primary network interface security groups or the security groups in ENIConfig
currentInstanceSecurityGroups []string
// subnetMask is the mask of the subnet CIDR block
subnetMask string
subnetMask string
subnetV6Mask string
// deviceIndexes is the list of indexes used by the EC2 Instance
deviceIndexes []bool
// primaryENIGroups is the security group used by the primary network interface
Expand All @@ -70,7 +73,9 @@ type EC2Instance interface {
InstanceID() string
SubnetID() string
SubnetMask() string
SubnetV6Mask() string
SubnetCidrBlock() string
SubnetV6CidrBlock() string
PrimaryNetworkInterfaceID() string
CurrentInstanceSecurityGroups() []string
SetNewCustomNetworkingSpec(subnetID string, securityGroup []string)
Expand Down Expand Up @@ -110,8 +115,16 @@ func (i *ec2Instance) LoadDetails(ec2APIHelper api.EC2APIHelper) error {
i.instanceSubnetID, i.instanceID)
}
i.instanceSubnetCidrBlock = *instanceSubnet.CidrBlock

i.subnetMask = strings.Split(i.instanceSubnetCidrBlock, "/")[1]
// Cache IPv6 CIDR block if one is present
for _, v6CidrBlock := range instanceSubnet.Ipv6CidrBlockAssociationSet {
if v6CidrBlock.Ipv6CidrBlock != nil {
i.instanceSubnetV6CidrBlock = *v6CidrBlock.Ipv6CidrBlock
i.subnetV6Mask = strings.Split(i.instanceSubnetV6CidrBlock, "/")[1]
break
}
}

i.instanceType = *instance.InstanceType
limits, ok := vpc.Limits[i.instanceType]
if !ok {
Expand Down Expand Up @@ -178,6 +191,13 @@ func (i *ec2Instance) SubnetCidrBlock() string {
return i.currentSubnetCIDRBlock
}

func (i *ec2Instance) SubnetV6CidrBlock() string {
i.lock.RLock()
defer i.lock.RUnlock()

return i.currentSubnetV6CIDRBlock
}

// Name returns the name of the node
func (i *ec2Instance) Name() string {
return i.name
Expand Down Expand Up @@ -231,6 +251,13 @@ func (i *ec2Instance) SubnetMask() string {
return i.subnetMask
}

func (i *ec2Instance) SubnetV6Mask() string {
i.lock.Lock()
defer i.lock.Unlock()

return i.subnetV6Mask
}

// SetNewCustomNetworkingSpec updates the subnet ID and subnet CIDR block for the instance
func (i *ec2Instance) SetNewCustomNetworkingSpec(subnet string, securityGroups []string) {
i.lock.Lock()
Expand Down Expand Up @@ -271,12 +298,14 @@ func (i *ec2Instance) updateCurrentSubnetAndCidrBlock(ec2APIHelper api.EC2APIHel
}
i.currentSubnetID = i.newCustomNetworkingSubnetID
i.currentSubnetCIDRBlock = *customSubnet.CidrBlock
// NOTE: IPv6 does not support custom networking
}
} else {
// Custom networking in not being used, point to the primary network interface security group and
// subnet details
i.currentSubnetID = i.instanceSubnetID
i.currentSubnetCIDRBlock = i.instanceSubnetCidrBlock
i.currentSubnetV6CIDRBlock = i.instanceSubnetV6CidrBlock
i.currentInstanceSecurityGroups = i.primaryENISecurityGroups
}

Expand Down
25 changes: 18 additions & 7 deletions pkg/provider/branch/trunk/trunk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,14 @@ type ENIDetails struct {
ID string `json:"eniId"`
// MacAdd is the MAC address of the network interface
MACAdd string `json:"ifAddress"`
// BranchIp is the primary IP of the branch Network interface
// IPv4 and/or IPv6 address assigned to the branch Network interface
IPV4Addr string `json:"privateIp"`
IPV6Addr string `json:"ipv6Addr"`
// VlanId is the VlanId of the branch network interface
VlanID int `json:"vlanId"`
// SubnetCIDR is the CIDR block of the subnet
SubnetCIDR string `json:"subnetCidr"`
SubnetCIDR string `json:"subnetCidr"`
SubnetV6CIDR string `json:"subnetV6Cidr"`
// deletionTimeStamp is the time when the pod was marked deleted.
deletionTimeStamp time.Time
// deleteRetryCount is the
Expand Down Expand Up @@ -358,9 +360,17 @@ func (t *trunkENI) CreateAndAssociateBranchENIs(pod *v1.Pod, securityGroups []st
break
}

// Branch ENI can have an IPv4 address, IPv6 address, or both
var v4Addr, v6Addr string
if nwInterface.PrivateIpAddress != nil {
v4Addr = *nwInterface.PrivateIpAddress
}
if nwInterface.Ipv6Address != nil {
v6Addr = *nwInterface.Ipv6Address
}
newENI := &ENIDetails{ID: *nwInterface.NetworkInterfaceId, MACAdd: *nwInterface.MacAddress,
IPV4Addr: *nwInterface.PrivateIpAddress, SubnetCIDR: t.instance.SubnetCidrBlock(), VlanID: vlanID}

IPV4Addr: v4Addr, IPV6Addr: v6Addr, SubnetCIDR: t.instance.SubnetCidrBlock(),
SubnetV6CIDR: t.instance.SubnetV6CidrBlock(), VlanID: vlanID}
newENIs = append(newENIs, newENI)

// Associate Branch to trunk
Expand Down Expand Up @@ -521,9 +531,10 @@ func (t *trunkENI) GetBranchInterfacesFromEC2() (eniDetails []*ENIDetails, err e
// For each association build the map of branch ENIs with the interface id and the vlan id
for _, association := range associations {
eniDetail := &ENIDetails{
ID: *association.BranchInterfaceId,
VlanID: int(*association.VlanId),
SubnetCIDR: t.instance.SubnetCidrBlock(),
ID: *association.BranchInterfaceId,
VlanID: int(*association.VlanId),
SubnetCIDR: t.instance.SubnetCidrBlock(),
SubnetV6CIDR: t.instance.SubnetV6CidrBlock(),
}
eniDetails = append(eniDetails, eniDetail)
}
Expand Down
56 changes: 35 additions & 21 deletions pkg/provider/branch/trunk/trunk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ var (
InstanceType = "c5.xlarge"
SubnetId = "subnet-00000000000000000"
SubnetCidrBlock = "192.168.0.0/16"
SubnetV6CidrBlock = "2600::/64"
NodeName = "test-node"
FakeInstance = ec2.NewEC2Instance(NodeName, InstanceId, config.OSLinux)
InstanceSecurityGroup = []string{"sg-1", "sg-2"}
Expand All @@ -56,9 +57,9 @@ var (
UID: MockPodUID1,
Name: MockPodName1,
Namespace: MockPodNamespace1,
Annotations: map[string]string{config.ResourceNamePodENI: "[{\"eniId\":\"eni-00000000000000000\",\"ifAddress\":\"FF:FF:FF:FF:FF:FF\",\"privateIp\":\"192.168.0.15\"" +
",\"vlanId\":1,\"subnetCidr\":\"192.168.0.0/16\"},{\"eniId\":\"eni-00000000000000001\",\"ifAddress\":\"" +
"FF:FF:FF:FF:FF:F9\",\"privateIp\":\"192.168.0.16\",\"vlanId\":2,\"subnetCidr\":\"192.168.0.0/16\"}]"}},
Annotations: map[string]string{config.ResourceNamePodENI: "[{\"eniId\":\"eni-00000000000000000\",\"ifAddress\":\"FF:FF:FF:FF:FF:FF\",\"privateIp\":\"192.168.0.15\"," +
"\"ipv6Addr\":\"2600::\",\"vlanId\":1,\"subnetCidr\":\"192.168.0.0/16\",\"subnetV6Cidr\":\"2600::/64\"},{\"eniId\":\"eni-00000000000000001\",\"ifAddress\":\"" +
"FF:FF:FF:FF:FF:F9\",\"privateIp\":\"192.168.0.16\",\"ipv6Addr\":\"2600::1\",\"vlanId\":2,\"subnetCidr\":\"192.168.0.0/16\",\"subnetV6Cidr\":\"2600::/64\"}]"}},
Spec: v1.PodSpec{NodeName: NodeName},
Status: v1.PodStatus{},
}
Expand Down Expand Up @@ -88,17 +89,20 @@ var (
SecurityGroups = []string{SecurityGroup1, SecurityGroup2}

// Branch Interface 1
Branch1Id = "eni-00000000000000000"
MacAddr1 = "FF:FF:FF:FF:FF:FF"
BranchIp1 = "192.168.0.15"
VlanId1 = 1
Branch1Id = "eni-00000000000000000"
MacAddr1 = "FF:FF:FF:FF:FF:FF"
BranchIp1 = "192.168.0.15"
BranchV6Ip1 = "2600::"
VlanId1 = 1

EniDetails1 = &ENIDetails{
ID: Branch1Id,
MACAdd: MacAddr1,
IPV4Addr: BranchIp1,
VlanID: VlanId1,
SubnetCIDR: SubnetCidrBlock,
ID: Branch1Id,
MACAdd: MacAddr1,
IPV4Addr: BranchIp1,
IPV6Addr: BranchV6Ip1,
VlanID: VlanId1,
SubnetCIDR: SubnetCidrBlock,
SubnetV6CIDR: SubnetV6CidrBlock,
}

branchENIs1 = []*ENIDetails{EniDetails1}
Expand All @@ -107,26 +111,31 @@ var (
MacAddress: &MacAddr1,
NetworkInterfaceId: &Branch1Id,
PrivateIpAddress: &BranchIp1,
Ipv6Address: &BranchV6Ip1,
}

// Branch Interface 2
Branch2Id = "eni-00000000000000001"
MacAddr2 = "FF:FF:FF:FF:FF:F9"
BranchIp2 = "192.168.0.16"
VlanId2 = 2
Branch2Id = "eni-00000000000000001"
MacAddr2 = "FF:FF:FF:FF:FF:F9"
BranchIp2 = "192.168.0.16"
BranchV6Ip2 = "2600::1"
VlanId2 = 2

EniDetails2 = &ENIDetails{
ID: Branch2Id,
MACAdd: MacAddr2,
IPV4Addr: BranchIp2,
VlanID: VlanId2,
SubnetCIDR: SubnetCidrBlock,
ID: Branch2Id,
MACAdd: MacAddr2,
IPV4Addr: BranchIp2,
IPV6Addr: BranchV6Ip2,
VlanID: VlanId2,
SubnetCIDR: SubnetCidrBlock,
SubnetV6CIDR: SubnetV6CidrBlock,
}

BranchInterface2 = &awsEc2.NetworkInterface{
MacAddress: &MacAddr2,
NetworkInterfaceId: &Branch2Id,
PrivateIpAddress: &BranchIp2,
Ipv6Address: &BranchV6Ip2,
}

branchENIs2 = []*ENIDetails{EniDetails2}
Expand Down Expand Up @@ -361,6 +370,7 @@ func TestTrunkENI_GetBranchInterfacesFromEC2(t *testing.T) {

ec2APIHelper.EXPECT().DescribeTrunkInterfaceAssociation(&trunkId).Return(trunkAssociationsBranch1And2, nil)
mockInstance.EXPECT().SubnetCidrBlock().Return(SubnetCidrBlock).Times(2)
mockInstance.EXPECT().SubnetV6CidrBlock().Return(SubnetV6CidrBlock).Times(2)

eniDetails, err := trunkENI.GetBranchInterfacesFromEC2()

Expand Down Expand Up @@ -778,6 +788,7 @@ func TestTrunkENI_CreateAndAssociateBranchENIs(t *testing.T) {
mockInstance.EXPECT().Type().Return(InstanceType)
mockInstance.EXPECT().SubnetID().Return(SubnetId).Times(2)
mockInstance.EXPECT().SubnetCidrBlock().Return(SubnetCidrBlock).Times(2)
mockInstance.EXPECT().SubnetV6CidrBlock().Return(SubnetV6CidrBlock).Times(2)

mockEC2APIHelper.EXPECT().CreateNetworkInterface(&BranchEniDescription, &SubnetId, SecurityGroups,
vlan1Tag, nil, nil).Return(BranchInterface1, nil)
Expand Down Expand Up @@ -810,6 +821,7 @@ func TestTrunkENI_CreateAndAssociateBranchENIs_InstanceSecurityGroup(t *testing.
mockInstance.EXPECT().Type().Return(InstanceType)
mockInstance.EXPECT().SubnetID().Return(SubnetId).Times(2)
mockInstance.EXPECT().SubnetCidrBlock().Return(SubnetCidrBlock).Times(2)
mockInstance.EXPECT().SubnetV6CidrBlock().Return(SubnetV6CidrBlock).Times(2)
mockInstance.EXPECT().CurrentInstanceSecurityGroups().Return(InstanceSecurityGroup)

mockEC2APIHelper.EXPECT().CreateNetworkInterface(&BranchEniDescription, &SubnetId, InstanceSecurityGroup,
Expand Down Expand Up @@ -843,6 +855,7 @@ func TestTrunkENI_CreateAndAssociateBranchENIs_ErrorAssociate(t *testing.T) {
mockInstance.EXPECT().Type().Return(InstanceType)
mockInstance.EXPECT().SubnetID().Return(SubnetId).Times(2)
mockInstance.EXPECT().SubnetCidrBlock().Return(SubnetCidrBlock).Times(2)
mockInstance.EXPECT().SubnetV6CidrBlock().Return(SubnetV6CidrBlock).Times(2)

gomock.InOrder(
mockEC2APIHelper.EXPECT().CreateNetworkInterface(&BranchEniDescription, &SubnetId, SecurityGroups,
Expand Down Expand Up @@ -870,6 +883,7 @@ func TestTrunkENI_CreateAndAssociateBranchENIs_ErrorCreate(t *testing.T) {
mockInstance.EXPECT().Type().Return(InstanceType)
mockInstance.EXPECT().SubnetID().Return(SubnetId).Times(2)
mockInstance.EXPECT().SubnetCidrBlock().Return(SubnetCidrBlock).Times(1)
mockInstance.EXPECT().SubnetV6CidrBlock().Return(SubnetV6CidrBlock).Times(1)

gomock.InOrder(
mockEC2APIHelper.EXPECT().CreateNetworkInterface(&BranchEniDescription, &SubnetId, SecurityGroups, vlan1Tag,
Expand Down
4 changes: 2 additions & 2 deletions scripts/gen_mocks.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/node/mock_n
mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/utils/mock_k8shelper.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils SecurityGroupForPodsAPI
# package pool mocks
mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/pool/mock_pool.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool Pool
# package resource maocks
# package resource mocks
mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/resource/mock_resources.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource ResourceManager
# package condition maocks
# package condition mocks
mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/condition/mock_condition.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition Conditions

0 comments on commit 86d7623

Please sign in to comment.