evpn: scope mac-mobility handling to MAC-VRF of the route

The mac mobility code used lookup tables to find the routes to the MAC
address. It only considered the MAC itself, and not the associated
MAC-VRF. This led to the complexity still being way too high when one
MAC is present in all MAC-VRFs, like a virtual router or an anycast
gateway.

This patch now considers the MAC-VRF for a route by looking up its route
targets, and considers them all if multiple are present. This way, the
lookup for routes that will participate in mac-mobility will be limited
to the MAC-VRF. This allows to not blow up the CPU when a MAC address is
present in a lot of MAC-VRFs.

The biggest behavioral change is now how mac-mobility is sharded. For
example, it can now have the same MAC in multiple MAC-VRFs on different
PEs. This is quite useful in highly multi-tenant environments where
control on the MAC addresses is nigh impossible. This is a net fix of
the behavior where MAC-VRFs cannot affect each other. For reference,
FRRouting does scope Mac Mobility to the MAC-VRF.

But the main reason this was done is to fix the quadratic complexity
that comes when there are many times the same MAC in many different
MAC-VRFs, by not needing to iterate through all the routes for other
unrelated MAC-VRFs.

A quick benchmark was done by adding type-2 routes through the gRPC API.
For 20k routes, on my laptop (Ryzen 5 PRO 4650U):

- without this patch
  * clean run: 228 seconds (3min 48s), starting around 1200 routes/s and
    decreasing (non linearily) to less than 50 routes/s
  * rerun a second time: 424 seconds (7min 4s), stuck at around 50
    routes/s
- with this patch, regardless of the whether the process is empty or
  not: 7.7s seconds, at 2700 routes/s the whole time

It is to be noted that, without the patch, the whole route ingestion
will be throttled by type-2 route ingestion.

internal/pkg/table/path.go

@@ -899,6 +899,16 @@ func (path *Path) SetExtCommunities(exts []bgp.ExtendedCommunityInterface, doRep
 	}
 }
 
+func (path *Path) GetRouteTargets() []bgp.ExtendedCommunityInterface {
+	rts := make([]bgp.ExtendedCommunityInterface, 0)
+	for _, ec := range path.GetExtCommunities() {
+		if t, st := ec.GetTypes(); t <= bgp.EC_TYPE_TRANSITIVE_FOUR_OCTET_AS_SPECIFIC && st == bgp.EC_SUBTYPE_ROUTE_TARGET {
+			rts = append(rts, ec)
+		}
+	}
+	return rts
+}
+
 func (path *Path) GetLargeCommunities() []*bgp.LargeCommunity {
 	if a := path.getPathAttr(bgp.BGP_ATTR_TYPE_LARGE_COMMUNITY); a != nil {
 		v := a.(*bgp.PathAttributeLargeCommunities).Values

internal/pkg/table/table.go

@@ -57,8 +57,8 @@ type Table struct {
 	routeFamily  bgp.RouteFamily
 	destinations map[string]*Destination
 	logger       log.Logger
-	// index of evpn prefixes with paths to a specific MAC
-	// this is a map[MAC address]map[prefix]struct{}
+	// index of evpn prefixes with paths to a specific MAC in a MAC-VRF
+	// this is a map[rt, MAC address]map[prefix]struct{}
 	// this holds a map for a set of prefixes.
 	macIndex map[string]map[string]struct{}
 }
@@ -146,12 +146,15 @@ func (t *Table) deleteDest(dest *Destination) {
 
 	if nlri, ok := dest.nlri.(*bgp.EVPNNLRI); ok {
 		if macadv, ok := nlri.RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute); ok {
-			mac := *(*string)(unsafe.Pointer(&macadv.MacAddress))
-			key := t.tableKey(nlri)
-			if keys, ok := t.macIndex[mac]; ok {
-				delete(keys, key)
-				if len(keys) == 0 {
-					delete(t.macIndex, mac)
+			for _, path := range dest.knownPathList {
+				for _, ec := range path.GetRouteTargets() {
+					macKey := t.macKey(ec, macadv.MacAddress)
+					if keys, ok := t.macIndex[macKey]; ok {
+						delete(keys, t.tableKey(nlri))
+						if len(keys) == 0 {
+							delete(t.macIndex, macKey)
+						}
+					}
 				}
 			}
 		}
@@ -214,6 +217,32 @@ func (t *Table) getOrCreateDest(nlri bgp.AddrPrefixInterface, size int) *Destina
 	return dest
 }
 
+func (t *Table) update(newPath *Path) *Update {
+	t.validatePath(newPath)
+	dst := t.getOrCreateDest(newPath.GetNlri(), 64)
+	u := dst.Calculate(t.logger, newPath)
+
+	if len(dst.knownPathList) == 0 {
+		t.deleteDest(dst)
+		return u
+	}
+
+	if nlri, ok := newPath.GetNlri().(*bgp.EVPNNLRI); ok {
+		if macadv, ok := nlri.RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute); ok {
+			tableKey := t.tableKey(nlri)
+			for _, ec := range newPath.GetRouteTargets() {
+				macKey := t.macKey(ec, macadv.MacAddress)
+				if _, ok := t.macIndex[macKey]; !ok {
+					t.macIndex[macKey] = make(map[string]struct{})
+				}
+				t.macIndex[macKey][tableKey] = struct{}{}
+			}
+		}
+	}
+
+	return u
+}
+
 func (t *Table) GetDestinations() map[string]*Destination {
 	return t.destinations
 }
@@ -392,16 +421,19 @@ func (t *Table) GetMUPDestinationsWithRouteType(p string) ([]*Destination, error
 }
 
 func (t *Table) setDestination(dst *Destination) {
-	t.destinations[t.tableKey(dst.nlri)] = dst
+	tableKey := t.tableKey(dst.nlri)
+	t.destinations[tableKey] = dst
 
 	if nlri, ok := dst.nlri.(*bgp.EVPNNLRI); ok {
 		if macadv, ok := nlri.RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute); ok {
-			mac := *(*string)(unsafe.Pointer(&macadv.MacAddress))
-			key := t.tableKey(nlri)
-			if keys, ok := t.macIndex[mac]; ok {
-				keys[key] = struct{}{}
-			} else {
-				t.macIndex[mac] = map[string]struct{}{key: {}}
+			for _, path := range dst.knownPathList {
+				for _, ec := range path.GetRouteTargets() {
+					macKey := t.macKey(ec, macadv.MacAddress)
+					if _, ok := t.macIndex[macKey]; !ok {
+						t.macIndex[macKey] = make(map[string]struct{})
+					}
+					t.macIndex[macKey][tableKey] = struct{}{}
+				}
 			}
 		}
 	}
@@ -437,6 +469,12 @@ func (t *Table) tableKey(nlri bgp.AddrPrefixInterface) string {
 	return nlri.String()
 }
 
+func (t *Table) macKey(rt bgp.ExtendedCommunityInterface, mac net.HardwareAddr) string {
+	b, _ := rt.Serialize()
+	b = append(b, mac...)
+	return *(*string)(unsafe.Pointer(&b))
+}
+
 func (t *Table) Bests(id string, as uint32) []*Path {
 	paths := make([]*Path, 0, len(t.destinations))
 	for _, dst := range t.destinations {
@@ -467,9 +505,9 @@ func (t *Table) GetKnownPathList(id string, as uint32) []*Path {
 	return paths
 }
 
-func (t *Table) GetKnownPathListWithMac(id string, as uint32, mac net.HardwareAddr, onlyBest bool) []*Path {
+func (t *Table) GetKnownPathListWithMac(id string, as uint32, rt bgp.ExtendedCommunityInterface, mac net.HardwareAddr, onlyBest bool) []*Path {
 	var paths []*Path
-	if prefixes, ok := t.macIndex[*(*string)(unsafe.Pointer(&mac))]; ok {
+	if prefixes, ok := t.macIndex[t.macKey(rt, mac)]; ok {
 		for prefix := range prefixes {
 			if dst, ok := t.destinations[prefix]; ok {
 				if onlyBest {

internal/pkg/table/table_manager.go

@@ -188,17 +188,6 @@ func (manager *TableManager) DeleteVrf(name string) ([]*Path, error) {
 	return msgs, nil
 }
 
-func (manager *TableManager) update(newPath *Path) *Update {
-	t := manager.Tables[newPath.GetRouteFamily()]
-	t.validatePath(newPath)
-	dst := t.getOrCreateDest(newPath.GetNlri(), 64)
-	u := dst.Calculate(manager.logger, newPath)
-	if len(dst.knownPathList) == 0 {
-		t.deleteDest(dst)
-	}
-	return u
-}
-
 func (manager *TableManager) Update(newPath *Path) []*Update {
 	if newPath == nil || newPath.IsEOR() {
 		return nil
@@ -207,12 +196,12 @@ func (manager *TableManager) Update(newPath *Path) []*Update {
 	// Except for a special case with EVPN, we'll have one destination.
 	updates := make([]*Update, 0, 1)
 	family := newPath.GetRouteFamily()
-	if _, ok := manager.Tables[family]; ok {
-		updates = append(updates, manager.update(newPath))
+	if table, ok := manager.Tables[family]; ok {
+		updates = append(updates, table.update(newPath))
 
 		if family == bgp.RF_EVPN {
 			for _, p := range manager.handleMacMobility(newPath) {
-				updates = append(updates, manager.update(p))
+				updates = append(updates, table.update(p))
 			}
 		}
 	}
@@ -255,7 +244,17 @@ func (manager *TableManager) handleMacMobility(path *Path) []*Path {
 	}
 	e1, et1, m1, s1, i1 := f(path)
 
-	for _, path2 := range manager.GetPathListWithMac(GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, m1) {
+	// Extract the route targets to scope the lookup to the MAC-VRF with the MAC address.
+	// This will help large EVPN instances where a single MAC is present in a lot of MAC-VRFs (e.g.
+	// an anycast router).
+	// A route may have multiple route targets, to target multiple MAC-VRFs (e.g. in both an L2VNI
+	// and L3VNI in the VXLAN case).
+	var paths []*Path
+	for _, ec := range path.GetRouteTargets() {
+		paths = append(paths, manager.GetPathListWithMac(GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, ec, m1)...)
+	}
+
+	for _, path2 := range paths {
 		if !path2.IsLocal() || path2.GetNlri().(*bgp.EVPNNLRI).RouteType != bgp.EVPN_ROUTE_TYPE_MAC_IP_ADVERTISEMENT {
 			continue
 		}
@@ -326,10 +325,10 @@ func (manager *TableManager) GetPathList(id string, as uint32, rfList []bgp.Rout
 	return paths
 }
 
-func (manager *TableManager) GetPathListWithMac(id string, as uint32, rfList []bgp.RouteFamily, mac net.HardwareAddr) []*Path {
+func (manager *TableManager) GetPathListWithMac(id string, as uint32, rfList []bgp.RouteFamily, rt bgp.ExtendedCommunityInterface, mac net.HardwareAddr) []*Path {
 	var paths []*Path
 	for _, t := range manager.tables(rfList...) {
-		paths = append(paths, t.GetKnownPathListWithMac(id, as, mac, false)...)
+		paths = append(paths, t.GetKnownPathListWithMac(id, as, rt, mac, false)...)
 	}
 	return paths
 }

pkg/server/server.go

@@ -2164,8 +2164,10 @@ func (s *BgpServer) fixupApiPath(vrfId string, pathList []*table.Path) error {
 			switch r := nlri.RouteTypeData.(type) {
 			case *bgp.EVPNMacIPAdvertisementRoute:
 				// MAC Mobility Extended Community
-				mac := path.GetNlri().(*bgp.EVPNNLRI).RouteTypeData.(*bgp.EVPNMacIPAdvertisementRoute).MacAddress
-				paths := s.globalRib.GetPathListWithMac(table.GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, mac)
+				var paths []*table.Path
+				for _, ec := range path.GetRouteTargets() {
+					paths = append(paths, s.globalRib.GetPathListWithMac(table.GLOBAL_RIB_NAME, 0, []bgp.RouteFamily{bgp.RF_EVPN}, ec, r.MacAddress)...)
+				}
 				if m := getMacMobilityExtendedCommunity(r.ETag, r.MacAddress, paths); m != nil {
 					pm := getMacMobilityExtendedCommunity(r.ETag, r.MacAddress, []*table.Path{path})
 					if pm == nil {