refine GetSiblingNumaInfo

cmd/katalyst-agent/app/options/global/base.go

@@ -69,6 +69,7 @@ type BaseOptions struct {
 	// configurations for machine-info
 	MachineNetMultipleNS                             bool
 	MachineNetNSDirAbsPath                           string
+	MachineSiblingNumaMaxDistance                    int
 	MachineSiblingNumaMemoryBandwidthCapacity        resource.QuantityValue
 	MachineSiblingNumaMemoryBandwidthAllocatableRate float64
 }
@@ -143,6 +144,8 @@ func (o *BaseOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 	fs.StringVar(&o.MachineNetNSDirAbsPath, "machine-net-ns-dir", o.MachineNetNSDirAbsPath,
 		"if set as true, we should collect network interfaces from multiple ns")
 
+	fs.IntVar(&o.MachineSiblingNumaMaxDistance, "machine-sibling-numa-max-distance", o.MachineSiblingNumaMaxDistance,
+		"The maximum distance between sibling NUMA nodes. If not set, the maximum distance defaults to the distance to itself.")
 	fs.Var(&o.MachineSiblingNumaMemoryBandwidthCapacity, "machine-sibling-numa-memory-bandwidth-capacity",
 		"if set the sibling numa memory bandwidth capacity, the per memory bandwidth capacity and allocatable will be reported to numa zone of cnr")
 	fs.Float64Var(&o.MachineSiblingNumaMemoryBandwidthAllocatableRate, "machine-sibling-numa-memory-bandwidth-allocatable-rate", o.MachineSiblingNumaMemoryBandwidthAllocatableRate,
@@ -163,6 +166,7 @@ func (o *BaseOptions) ApplyTo(c *global.BaseConfiguration) error {
 
 	c.NetMultipleNS = o.MachineNetMultipleNS
 	c.NetNSDirAbsPath = o.MachineNetNSDirAbsPath
+	c.SiblingNumaMaxDistance = o.MachineSiblingNumaMaxDistance
 	c.SiblingNumaMemoryBandwidthCapacity = o.MachineSiblingNumaMemoryBandwidthCapacity.Quantity.Value()
 	c.SiblingNumaMemoryBandwidthAllocatableRate = o.MachineSiblingNumaMemoryBandwidthAllocatableRate
 

pkg/config/agent/global/base.go

@@ -47,6 +47,11 @@ type MachineInfoConfiguration struct {
 	NetMultipleNS   bool
 	NetNSDirAbsPath string
 
+	// SiblingNumaMaxDistance represents the maximum distance between sibling NUMA nodes.
+	// These sibling NUMA nodes have the smallest distance to each other, except for the
+	// distance to themselves.
+	SiblingNumaMaxDistance int
+
 	// SiblingNumaMemoryBandwidthCapacity is the max capacity of memory bandwidth can share
 	// among sibling NUMAs, and SiblingNumaMemoryBandwidthAllocatableRate is the rate of
 	// the allocatable to the capacity

pkg/util/machine/topology.go

@@ -18,13 +18,15 @@ package machine
 
 import (
 	"fmt"
+	"math"
 
 	info "github.com/google/cadvisor/info/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 
 	"github.com/kubewharf/katalyst-core/pkg/config/agent/global"
+	"github.com/kubewharf/katalyst-core/pkg/util/general"
 )
 
 // NUMANodeInfo is a map from NUMANode ID to a list of
@@ -525,22 +527,29 @@ func GetSiblingNumaInfo(conf *global.MachineInfoConfiguration,
 
 	for numaID, distanceMap := range numaDistanceMap {
 		var selfNumaDistance int
+		// calculate self NUMA distance and the minimum cross-NUMA distance.
+		minCrossNumaDistance := math.MaxInt
 		for _, distance := range distanceMap {
 			if distance.NumaID == numaID {
 				selfNumaDistance = distance.Distance
-				break
+			} else {
+				minCrossNumaDistance = general.Min(distance.Distance, minCrossNumaDistance)
 			}
 		}
+		// the sibling NUMA distance must be no smaller than the distance to itself
+		// and no larger than the minimum cross-NUMA distance.
+		siblingNumaDistance := general.Min(general.Max(selfNumaDistance, conf.SiblingNumaMaxDistance),
+			minCrossNumaDistance)
 
 		siblingSet := sets.NewInt()
 		for _, distance := range distanceMap {
 			if distance.NumaID == numaID {
 				continue
 			}
 
-			// the distance between two different NUMAs is equal to the distance between
-			// it and itself are siblings each other
-			if distance.Distance == selfNumaDistance {
+			// the distance between two different NUMAs is equal to the sibling
+			// numa distance
+			if distance.Distance == siblingNumaDistance {
 				siblingSet.Insert(distance.NumaID)
 			}
 		}