Skip to content

Commit

Permalink
ipv6 security groups for pods support
Browse files Browse the repository at this point in the history
  • Loading branch information
jdn5126 committed Nov 27, 2023
1 parent d49c8a3 commit 4bcdeef
Show file tree
Hide file tree
Showing 19 changed files with 1,528 additions and 286 deletions.
15 changes: 10 additions & 5 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,23 @@ aws-k8s-agent
aws-cni
aws-vpc-cni
aws-vpc-cni-init
cni-metrics-helper
egress-cni
grpc-health-probe
portmap
verify-aws
verify-network
# Ignore generated files
*~
*.swp
.idea/
*.iml
.DS_Store
portmap
grpc-health-probe
cni-metrics-helper
coverage.txt
cmd/egress-cni-plugin/egress-plugin.log
# Ignore build files
core-plugins/
build/
vendor
egress-cni
vendor/
# Unignore charts directory as its confilcts with `aws-vpc-cni` binary ignore rule
!charts/**
29 changes: 28 additions & 1 deletion cmd/aws-vpc-cni-init/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,10 @@ func configureIPv6Settings(procSys procsyswrapper.ProcSys, primaryIF string) err
// Check if IPv6 egress support is enabled in IPv4 cluster.
ipv6EgressEnabled := utils.GetBoolAsStringEnvVar(envEgressV6, defaultEnableIPv6Egress)
if enableIPv6 || ipv6EgressEnabled {
// For IPv6, the following sysctls are set:
// 1. forwarding defaults to 1
// 2. accept_ra defaults to 2
// 3. accept_redirects defaults to 1
entry := "net/ipv6/conf/all/forwarding"
err = procSys.Set(entry, "1")
if err != nil {
Expand All @@ -125,10 +129,33 @@ func configureIPv6Settings(procSys procsyswrapper.ProcSys, primaryIF string) err
val, _ := procSys.Get(entry)
log.Infof("Updated %s to %s", entry, val)

// accept_ra must be set to 2 so that RA routes are installed by the kernel on secondary ENIs
// For IPv6, this setting must be inherited by the trunk ENI. It must be set here as IPAMD does
// not have permission to set sysctl values.
entry = "net/ipv6/conf/default/accept_ra"
err = procSys.Set(entry, "2")
if err != nil {
return errors.Wrap(err, "Failed to set IPv6 accept Router Advertisements to 2")
}
val, _ = procSys.Get(entry)
log.Infof("Updated %s to %s", entry, val)

entry = "net/ipv6/conf/default/accept_redirects"
err = procSys.Set(entry, "1")
if err != nil {
return errors.Wrap(err, "Failed to enable IPv6 accept redirects")
}
val, _ = procSys.Get(entry)
log.Infof("Updated %s to %s", entry, val)

// For the primary ENI in IPv6, sysctls are set to:
// 1. forwarding=1
// 2. accept_ra=2
// 3. accept_redirects=1
entry = "net/ipv6/conf/" + primaryIF + "/accept_ra"
err = procSys.Set(entry, "2")
if err != nil {
return errors.Wrap(err, "Failed to enable IPv6 accept_ra")
return errors.Wrap(err, "Failed to enable IPv6 accept_ra on primary ENI")
}
val, _ = procSys.Get(entry)
log.Infof("Updated %s to %s", entry, val)
Expand Down
73 changes: 64 additions & 9 deletions cmd/routed-eni-cni-plugin/driver/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,9 @@ func (n *linuxNetwork) SetupBranchENIPodNetwork(hostVethName string, contVethNam
oldFromHostVethRule := n.netLink.NewRule()
oldFromHostVethRule.IifName = hostVethName
oldFromHostVethRule.Priority = networkutils.VlanRulePriority
if v6Addr != nil {
oldFromHostVethRule.Family = unix.AF_INET6
}
if err := networkutils.NetLinkRuleDelAll(n.netLink, oldFromHostVethRule); err != nil {
return errors.Wrapf(err, "SetupBranchENIPodNetwork: failed to delete hostVeth rule for %s", hostVethName)
}
Expand Down Expand Up @@ -328,9 +331,13 @@ func (n *linuxNetwork) TeardownBranchENIPodNetwork(containerAddr *net.IPNet, vla
return errors.Wrapf(err, "TeardownBranchENIPodNetwork: failed to teardown vlan")
}

ipFamily := unix.AF_INET
if containerAddr.IP.To4() == nil {
ipFamily = unix.AF_INET6
}
// to handle the migration between different enforcingMode, we try to clean up rules under both mode since the pod might be setup with a different mode.
rtTable := vlanID + 100
if err := n.teardownIIFBasedContainerRouteRules(rtTable, log); err != nil {
if err := n.teardownIIFBasedContainerRouteRules(rtTable, ipFamily, log); err != nil {
return errors.Wrapf(err, "TeardownBranchENIPodNetwork: unable to teardown IIF based container routes and rules")
}
if err := n.teardownIPBasedContainerRouteRules(containerAddr, rtTable, log); err != nil {
Expand Down Expand Up @@ -360,20 +367,29 @@ func (n *linuxNetwork) setupVeth(hostVethName string, contVethName string, netns
return nil, errors.Wrapf(err, "failed to find hostVeth %s", hostVethName)
}

// For IPv6, host veth sysctls must be set to:
// 1. accept_ra=0
// 2. accept_redirects=1
// 3. forwarding=0
if err := n.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", hostVethName), "0"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "failed to disable IPv6 router advertisements")
}
log.Debugf("Ignoring '%v' writing to accept_ra: Assuming kernel lacks IPv6 support", err)
}

if err := n.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_redirects", hostVethName), "0"); err != nil {
if err := n.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_redirects", hostVethName), "1"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "failed to disable IPv6 ICMP redirects")
}
log.Debugf("Ignoring '%v' writing to accept_redirects: Assuming kernel lacks IPv6 support", err)
}
log.Debugf("Successfully disabled IPv6 RA and ICMP redirects on hostVeth %s", hostVethName)
if err := n.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/forwarding", hostVethName), "0"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "failed to disable IPv6 forwarding")
}
log.Debugf("Ignoring '%v' writing to forwarding: Assuming kernel lacks IPv6 support", err)
}
log.Debugf("Successfully set IPv6 sysctls on hostVeth %s", hostVethName)

// Explicitly set the veth to UP state, because netlink doesn't always do that on all the platforms with net.FlagUp.
// veth won't get a link local address unless it's set to UP state.
Expand All @@ -400,12 +416,37 @@ func (n *linuxNetwork) setupVlan(vlanID int, eniMAC string, subnetGW string, par
return nil, errors.Wrapf(err, "failed to add vlan link %s", vlanLinkName)
}

// 3. bring up the vlan
// 3. Set IPv6 sysctls
// accept_ra=0
// accept_redirects=1
// forwarding=0
if err := n.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", vlanLinkName), "0"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "failed to disable IPv6 router advertisements")
}
log.Debugf("Ignoring '%v' writing to accept_ra: Assuming kernel lacks IPv6 support", err)
}

if err := n.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_redirects", vlanLinkName), "1"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "failed to enable IPv6 ICMP redirects")
}
log.Debugf("Ignoring '%v' writing to accept_redirects: Assuming kernel lacks IPv6 support", err)
}

if err := n.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/forwarding", vlanLinkName), "0"); err != nil {
if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "failed to disable IPv6 forwarding")
}
log.Debugf("Ignoring '%v' writing to forwarding: Assuming kernel lacks IPv6 support", err)
}

// 4. bring up the vlan
if err := n.netLink.LinkSetUp(vlanLink); err != nil {
return nil, errors.Wrapf(err, "failed to setUp vlan link %s", vlanLinkName)
}

// 4. create default routes for vlan
// 5. create default routes for vlan
routes := buildRoutesForVlan(rtTable, vlanLink.Index, net.ParseIP(subnetGW))
for _, r := range routes {
if err := n.netLink.RouteReplace(&r); err != nil {
Expand Down Expand Up @@ -511,6 +552,7 @@ func (n *linuxNetwork) teardownIPBasedContainerRouteRules(containerAddr *net.IPN
// traffic to container(iif hostVlan) will be routed via the specified rtTable.
// traffic from container(iif hostVeth) will be routed via the specified rtTable.
func (n *linuxNetwork) setupIIFBasedContainerRouteRules(hostVeth netlink.Link, containerAddr *net.IPNet, hostVlan netlink.Link, rtTable int, log logger.Logger) error {
isV6 := containerAddr.IP.To4() == nil
route := netlink.Route{
LinkIndex: hostVeth.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Expand All @@ -528,6 +570,9 @@ func (n *linuxNetwork) setupIIFBasedContainerRouteRules(hostVeth netlink.Link, c
fromHostVlanRule.IifName = hostVlan.Attrs().Name
fromHostVlanRule.Priority = networkutils.VlanRulePriority
fromHostVlanRule.Table = rtTable
if isV6 {
fromHostVlanRule.Family = unix.AF_INET6
}
if err := n.netLink.RuleAdd(fromHostVlanRule); err != nil && !networkutils.IsRuleExistsError(err) {
return errors.Wrapf(err, "unable to setup fromHostVlan rule, hostVlan=%s, rtTable=%v", hostVlan.Attrs().Name, rtTable)
}
Expand All @@ -537,6 +582,9 @@ func (n *linuxNetwork) setupIIFBasedContainerRouteRules(hostVeth netlink.Link, c
fromHostVethRule.IifName = hostVeth.Attrs().Name
fromHostVethRule.Priority = networkutils.VlanRulePriority
fromHostVethRule.Table = rtTable
if isV6 {
fromHostVethRule.Family = unix.AF_INET6
}
if err := n.netLink.RuleAdd(fromHostVethRule); err != nil && !networkutils.IsRuleExistsError(err) {
return errors.Wrapf(err, "unable to setup fromHostVeth rule, hostVeth=%s, rtTable=%v", hostVeth.Attrs().Name, rtTable)
}
Expand All @@ -545,10 +593,11 @@ func (n *linuxNetwork) setupIIFBasedContainerRouteRules(hostVeth netlink.Link, c
return nil
}

func (n *linuxNetwork) teardownIIFBasedContainerRouteRules(rtTable int, log logger.Logger) error {
func (n *linuxNetwork) teardownIIFBasedContainerRouteRules(rtTable int, family int, log logger.Logger) error {
rule := n.netLink.NewRule()
rule.Priority = networkutils.VlanRulePriority
rule.Table = rtTable
rule.Family = family

if err := networkutils.NetLinkRuleDelAll(n.netLink, rule); err != nil {
return errors.Wrapf(err, "failed to delete IIF based rules, rtTable=%v", rtTable)
Expand All @@ -560,17 +609,23 @@ func (n *linuxNetwork) teardownIIFBasedContainerRouteRules(rtTable int, log logg

// buildRoutesForVlan builds routes required for the vlan link.
func buildRoutesForVlan(vlanTableID int, vlanIndex int, gw net.IP) []netlink.Route {
maskLen := 32
zeroAddr := net.IPv4zero
if gw.To4() == nil {
maskLen = 128
zeroAddr = net.IPv6zero
}
return []netlink.Route{
// Add a direct link route for the pod vlan link only.
{
LinkIndex: vlanIndex,
Dst: &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)},
Dst: &net.IPNet{IP: gw, Mask: net.CIDRMask(maskLen, maskLen)},
Scope: netlink.SCOPE_LINK,
Table: vlanTableID,
},
{
LinkIndex: vlanIndex,
Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
Dst: &net.IPNet{IP: zeroAddr, Mask: net.CIDRMask(0, maskLen)},
Scope: netlink.SCOPE_UNIVERSE,
Gw: gw,
Table: vlanTableID,
Expand Down
Loading

0 comments on commit 4bcdeef

Please sign in to comment.