From c438a2c89703c2343c59a9292700d5d9656546bf Mon Sep 17 00:00:00 2001 From: Tuetuopay Date: Fri, 3 May 2024 14:12:51 +0200 Subject: [PATCH] evpn: scope mac-mobility handling to MAC-VRF of the route The mac mobility code used lookup tables to find the routes to the MAC address. It only considered the MAC itself, and not the associated MAC-VRF. This led to the complexity still being way too high when one MAC is present in all MAC-VRFs, like a virtual router or an anycast gateway. This patch now considers the MAC-VRF for a route by looking up its route targets, and considers them all if multiple are present. This way, the lookup for routes that will participate in mac-mobility will be limited to the MAC-VRF. This allows to not blow up the CPU when a MAC address is present in a lot of MAC-VRFs. The biggest behavioral change is now how mac-mobility is sharded. For example, it can now have the same MAC in multiple MAC-VRFs on different PEs. This is quite useful in highly multi-tenant environments where control on the MAC addresses is nigh impossible. This is a net fix of the behavior where MAC-VRFs cannot affect each other. For reference, FRRouting does scope Mac Mobility to the MAC-VRF. But the main reason this was done is to fix the quadratic complexity that comes when there are many times the same MAC in many different MAC-VRFs, by not needing to iterate through all the routes for other unrelated MAC-VRFs. A quick benchmark was done by adding type-2 routes through the gRPC API. For 20k routes, on my laptop (Ryzen 5 PRO 4650U): - without this patch * clean run: 228 seconds (3min 48s), starting around 1200 routes/s and decreasing (non linearily) to less than 50 routes/s * rerun a second time: 424 seconds (7min 4s), stuck at around 50 routes/s - with this patch, regardless of the whether the process is empty or not: 7.7s seconds, at 2700 routes/s the whole time It is to be noted that, without the patch, the whole route ingestion will be throttled by type-2 route ingestion. --- internal/pkg/table/path.go | 10 ++++ internal/pkg/table/table.go | 72 ++++++++++++++++++++++------- internal/pkg/table/table_manager.go | 33 +++++++------ pkg/server/server.go | 6 ++- 4 files changed, 85 insertions(+), 36 deletions(-) diff --git a/internal/pkg/table/path.go b/internal/pkg/table/path.go index dc3139452..60e9d1526 100644 --- a/internal/pkg/table/path.go +++ b/internal/pkg/table/path.go @@ -899,6 +899,16 @@ func (path *Path) SetExtCommunities(exts []bgp.ExtendedCommunityInterface, doRep } } +func (path *Path) GetRouteTargets() []bgp.ExtendedCommunityInterface { + rts := make([]bgp.ExtendedCommunityInterface, 0) + for _, ec := range path.GetExtCommunities() { + if t, st := ec.GetTypes(); t <= bgp.EC_TYPE_TRANSITIVE_FOUR_OCTET_AS_SPECIFIC && st == bgp.EC_SUBTYPE_ROUTE_TARGET { + rts = append(rts, ec) + } + } + return rts +} + func (path *Path) GetLargeCommunities() []*bgp.LargeCommunity { if a := path.getPathAttr(bgp.BGP_ATTR_TYPE_LARGE_COMMUNITY); a != nil { v := a.(*bgp.PathAttributeLargeCommunities).Values diff --git a/internal/pkg/table/table.go b/internal/pkg/table/table.go index 7bf5aa73c..1997b3c7b 100644 --- a/internal/pkg/table/table.go +++ b/internal/pkg/table/table.go @@ -57,8 +57,8 @@ type Table struct { routeFamily bgp.RouteFamily destinations map[string]*Destination logger log.Logger - // index of evpn prefixes with paths to a specific MAC - // this is a map[MAC address]map[prefix]struct{} + // index of evpn prefixes with paths to a specific MAC in a MAC-VRF + // this is a map[rt, MAC address]map[prefix]struct{} // this holds a map for a set of prefixes. macIndex map[string]map[string]struct{} } @@ -146,12 +146,15 @@ func (t *Table) deleteDest(dest *Destination) { if nlri, ok := dest.nlri.(*bgp.EVPNNLRI); ok { if macadv, ok := nlri.RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute); ok { - mac := *(*string)(unsafe.Pointer(&macadv.MacAddress)) - key := t.tableKey(nlri) - if keys, ok := t.macIndex[mac]; ok { - delete(keys, key) - if len(keys) == 0 { - delete(t.macIndex, mac) + for _, path := range dest.knownPathList { + for _, ec := range path.GetRouteTargets() { + macKey := t.macKey(ec, macadv.MacAddress) + if keys, ok := t.macIndex[macKey]; ok { + delete(keys, t.tableKey(nlri)) + if len(keys) == 0 { + delete(t.macIndex, macKey) + } + } } } } @@ -214,6 +217,32 @@ func (t *Table) getOrCreateDest(nlri bgp.AddrPrefixInterface, size int) *Destina return dest } +func (t *Table) update(newPath *Path) *Update { + t.validatePath(newPath) + dst := t.getOrCreateDest(newPath.GetNlri(), 64) + u := dst.Calculate(t.logger, newPath) + + if len(dst.knownPathList) == 0 { + t.deleteDest(dst) + return u + } + + if nlri, ok := newPath.GetNlri().(*bgp.EVPNNLRI); ok { + if macadv, ok := nlri.RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute); ok { + tableKey := t.tableKey(nlri) + for _, ec := range newPath.GetRouteTargets() { + macKey := t.macKey(ec, macadv.MacAddress) + if _, ok := t.macIndex[macKey]; !ok { + t.macIndex[macKey] = make(map[string]struct{}) + } + t.macIndex[macKey][tableKey] = struct{}{} + } + } + } + + return u +} + func (t *Table) GetDestinations() map[string]*Destination { return t.destinations } @@ -392,16 +421,19 @@ func (t *Table) GetMUPDestinationsWithRouteType(p string) ([]*Destination, error } func (t *Table) setDestination(dst *Destination) { - t.destinations[t.tableKey(dst.nlri)] = dst + tableKey := t.tableKey(dst.nlri) + t.destinations[tableKey] = dst if nlri, ok := dst.nlri.(*bgp.EVPNNLRI); ok { if macadv, ok := nlri.RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute); ok { - mac := *(*string)(unsafe.Pointer(&macadv.MacAddress)) - key := t.tableKey(nlri) - if keys, ok := t.macIndex[mac]; ok { - keys[key] = struct{}{} - } else { - t.macIndex[mac] = map[string]struct{}{key: {}} + for _, path := range dst.knownPathList { + for _, ec := range path.GetRouteTargets() { + macKey := t.macKey(ec, macadv.MacAddress) + if _, ok := t.macIndex[macKey]; !ok { + t.macIndex[macKey] = make(map[string]struct{}) + } + t.macIndex[macKey][tableKey] = struct{}{} + } } } } @@ -437,6 +469,12 @@ func (t *Table) tableKey(nlri bgp.AddrPrefixInterface) string { return nlri.String() } +func (t *Table) macKey(rt bgp.ExtendedCommunityInterface, mac net.HardwareAddr) string { + b, _ := rt.Serialize() + b = append(b, mac...) + return *(*string)(unsafe.Pointer(&b)) +} + func (t *Table) Bests(id string, as uint32) []*Path { paths := make([]*Path, 0, len(t.destinations)) for _, dst := range t.destinations { @@ -467,9 +505,9 @@ func (t *Table) GetKnownPathList(id string, as uint32) []*Path { return paths } -func (t *Table) GetKnownPathListWithMac(id string, as uint32, mac net.HardwareAddr, onlyBest bool) []*Path { +func (t *Table) GetKnownPathListWithMac(id string, as uint32, rt bgp.ExtendedCommunityInterface, mac net.HardwareAddr, onlyBest bool) []*Path { var paths []*Path - if prefixes, ok := t.macIndex[*(*string)(unsafe.Pointer(&mac))]; ok { + if prefixes, ok := t.macIndex[t.macKey(rt, mac)]; ok { for prefix := range prefixes { if dst, ok := t.destinations[prefix]; ok { if onlyBest { diff --git a/internal/pkg/table/table_manager.go b/internal/pkg/table/table_manager.go index dca893e5b..d25c02787 100644 --- a/internal/pkg/table/table_manager.go +++ b/internal/pkg/table/table_manager.go @@ -188,17 +188,6 @@ func (manager *TableManager) DeleteVrf(name string) ([]*Path, error) { return msgs, nil } -func (manager *TableManager) update(newPath *Path) *Update { - t := manager.Tables[newPath.GetRouteFamily()] - t.validatePath(newPath) - dst := t.getOrCreateDest(newPath.GetNlri(), 64) - u := dst.Calculate(manager.logger, newPath) - if len(dst.knownPathList) == 0 { - t.deleteDest(dst) - } - return u -} - func (manager *TableManager) Update(newPath *Path) []*Update { if newPath == nil || newPath.IsEOR() { return nil @@ -207,12 +196,12 @@ func (manager *TableManager) Update(newPath *Path) []*Update { // Except for a special case with EVPN, we'll have one destination. updates := make([]*Update, 0, 1) family := newPath.GetRouteFamily() - if _, ok := manager.Tables[family]; ok { - updates = append(updates, manager.update(newPath)) + if table, ok := manager.Tables[family]; ok { + updates = append(updates, table.update(newPath)) if family == bgp.RF_EVPN { for _, p := range manager.handleMacMobility(newPath) { - updates = append(updates, manager.update(p)) + updates = append(updates, table.update(p)) } } } @@ -255,7 +244,17 @@ func (manager *TableManager) handleMacMobility(path *Path) []*Path { } e1, et1, m1, s1, i1 := f(path) - for _, path2 := range manager.GetPathListWithMac(GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, m1) { + // Extract the route targets to scope the lookup to the MAC-VRF with the MAC address. + // This will help large EVPN instances where a single MAC is present in a lot of MAC-VRFs (e.g. + // an anycast router). + // A route may have multiple route targets, to target multiple MAC-VRFs (e.g. in both an L2VNI + // and L3VNI in the VXLAN case). + var paths []*Path + for _, ec := range path.GetRouteTargets() { + paths = append(paths, manager.GetPathListWithMac(GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, ec, m1)...) + } + + for _, path2 := range paths { if !path2.IsLocal() || path2.GetNlri().(*bgp.EVPNNLRI).RouteType != bgp.EVPN_ROUTE_TYPE_MAC_IP_ADVERTISEMENT { continue } @@ -326,10 +325,10 @@ func (manager *TableManager) GetPathList(id string, as uint32, rfList []bgp.Rout return paths } -func (manager *TableManager) GetPathListWithMac(id string, as uint32, rfList []bgp.RouteFamily, mac net.HardwareAddr) []*Path { +func (manager *TableManager) GetPathListWithMac(id string, as uint32, rfList []bgp.RouteFamily, rt bgp.ExtendedCommunityInterface, mac net.HardwareAddr) []*Path { var paths []*Path for _, t := range manager.tables(rfList...) { - paths = append(paths, t.GetKnownPathListWithMac(id, as, mac, false)...) + paths = append(paths, t.GetKnownPathListWithMac(id, as, rt, mac, false)...) } return paths } diff --git a/pkg/server/server.go b/pkg/server/server.go index a43ebe928..e8fa6d234 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -2164,8 +2164,10 @@ func (s *BgpServer) fixupApiPath(vrfId string, pathList []*table.Path) error { switch r := nlri.RouteTypeData.(type) { case *bgp.EVPNMacIPAdvertisementRoute: // MAC Mobility Extended Community - mac := path.GetNlri().(*bgp.EVPNNLRI).RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute).MacAddress - paths := s.globalRib.GetPathListWithMac(table.GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, mac) + var paths []*table.Path + for _, ec := range path.GetRouteTargets() { + paths = append(paths, s.globalRib.GetPathListWithMac(table.GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, ec, r.MacAddress)...) + } if m := getMacMobilityExtendedCommunity(r.ETag, r.MacAddress, paths); m != nil { pm := getMacMobilityExtendedCommunity(r.ETag, r.MacAddress, []*table.Path{path}) if pm == nil {