refine GetSiblingNumaInfo logic support sibling numa distance larger …

…than the distance to itself

cmd/katalyst-agent/app/options/global/base.go

@@ -69,6 +69,7 @@ type BaseOptions struct {
 	// configurations for machine-info
 	MachineNetMultipleNS                             bool
 	MachineNetNSDirAbsPath                           string
+	MachineSiblingNumaMaxDistance                    int
 	MachineSiblingNumaMemoryBandwidthCapacity        resource.QuantityValue
 	MachineSiblingNumaMemoryBandwidthAllocatableRate float64
 }
@@ -143,6 +144,8 @@ func (o *BaseOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 	fs.StringVar(&o.MachineNetNSDirAbsPath, "machine-net-ns-dir", o.MachineNetNSDirAbsPath,
 		"if set as true, we should collect network interfaces from multiple ns")
 
+	fs.IntVar(&o.MachineSiblingNumaMaxDistance, "machine-sibling-numa-max-distance", o.MachineSiblingNumaMaxDistance,
+		"The maximum distance between sibling NUMA nodes. If not set, the maximum distance defaults to the distance to itself.")
 	fs.Var(&o.MachineSiblingNumaMemoryBandwidthCapacity, "machine-sibling-numa-memory-bandwidth-capacity",
 		"if set the sibling numa memory bandwidth capacity, the per memory bandwidth capacity and allocatable will be reported to numa zone of cnr")
 	fs.Float64Var(&o.MachineSiblingNumaMemoryBandwidthAllocatableRate, "machine-sibling-numa-memory-bandwidth-allocatable-rate", o.MachineSiblingNumaMemoryBandwidthAllocatableRate,
@@ -163,6 +166,7 @@ func (o *BaseOptions) ApplyTo(c *global.BaseConfiguration) error {
 
 	c.NetMultipleNS = o.MachineNetMultipleNS
 	c.NetNSDirAbsPath = o.MachineNetNSDirAbsPath
+	c.SiblingNumaMaxDistance = o.MachineSiblingNumaMaxDistance
 	c.SiblingNumaMemoryBandwidthCapacity = o.MachineSiblingNumaMemoryBandwidthCapacity.Quantity.Value()
 	c.SiblingNumaMemoryBandwidthAllocatableRate = o.MachineSiblingNumaMemoryBandwidthAllocatableRate
 

pkg/config/agent/global/base.go

@@ -47,6 +47,11 @@ type MachineInfoConfiguration struct {
 	NetMultipleNS   bool
 	NetNSDirAbsPath string
 
+	// SiblingNumaMaxDistance represents the maximum distance between sibling NUMA nodes.
+	// These sibling NUMA nodes have the smallest distance to each other, except for the
+	// distance to themselves.
+	SiblingNumaMaxDistance int
+
 	// SiblingNumaMemoryBandwidthCapacity is the max capacity of memory bandwidth can share
 	// among sibling NUMAs, and SiblingNumaMemoryBandwidthAllocatableRate is the rate of
 	// the allocatable to the capacity

pkg/util/machine/topology.go

@@ -18,13 +18,15 @@ package machine
 
 import (
 	"fmt"
+	"math"
 
 	info "github.com/google/cadvisor/info/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 
 	"github.com/kubewharf/katalyst-core/pkg/config/agent/global"
+	"github.com/kubewharf/katalyst-core/pkg/util/general"
 )
 
 // NUMANodeInfo is a map from NUMANode ID to a list of
@@ -525,22 +527,29 @@ func GetSiblingNumaInfo(conf *global.MachineInfoConfiguration,
 
 	for numaID, distanceMap := range numaDistanceMap {
 		var selfNumaDistance int
+		// calculate self NUMA distance and the minimum cross-NUMA distance.
+		minCrossNumaDistance := math.MaxInt
 		for _, distance := range distanceMap {
 			if distance.NumaID == numaID {
 				selfNumaDistance = distance.Distance
-				break
+			} else {
+				minCrossNumaDistance = general.Min(distance.Distance, minCrossNumaDistance)
 			}
 		}
+		// the sibling NUMA distance must be no smaller than the distance to itself
+		// and no larger than the minimum cross-NUMA distance.
+		siblingNumaDistance := general.Min(general.Max(selfNumaDistance, conf.SiblingNumaMaxDistance),
+			minCrossNumaDistance)
 
 		siblingSet := sets.NewInt()
 		for _, distance := range distanceMap {
 			if distance.NumaID == numaID {
 				continue
 			}
 
-			// the distance between two different NUMAs is equal to the distance between
-			// it and itself are siblings each other
-			if distance.Distance == selfNumaDistance {
+			// the distance between two different NUMAs is equal to the sibling
+			// numa distance
+			if distance.Distance == siblingNumaDistance {
 				siblingSet.Insert(distance.NumaID)
 			}
 		}

pkg/util/machine/topology_test.go

@@ -326,6 +326,110 @@ func TestGetSiblingNumaInfo(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "test for with sibling with distance larger than 10",
+			args: args{
+				conf: &global.MachineInfoConfiguration{
+					SiblingNumaMemoryBandwidthAllocatableRate: 0.8,
+					SiblingNumaMaxDistance:                    11,
+					SiblingNumaMemoryBandwidthCapacity:        10,
+				},
+				numaDistanceMap: map[int][]NumaDistanceInfo{
+					0: {
+						{
+							NumaID:   0,
+							Distance: 10,
+						},
+						{
+							NumaID:   1,
+							Distance: 11,
+						},
+						{
+							NumaID:   2,
+							Distance: 32,
+						},
+						{
+							NumaID:   3,
+							Distance: 32,
+						},
+					},
+					1: {
+						{
+							NumaID:   0,
+							Distance: 11,
+						},
+						{
+							NumaID:   1,
+							Distance: 10,
+						},
+						{
+							NumaID:   2,
+							Distance: 32,
+						},
+						{
+							NumaID:   3,
+							Distance: 32,
+						},
+					},
+					2: {
+						{
+							NumaID:   0,
+							Distance: 32,
+						},
+						{
+							NumaID:   1,
+							Distance: 32,
+						},
+						{
+							NumaID:   2,
+							Distance: 10,
+						},
+						{
+							NumaID:   3,
+							Distance: 11,
+						},
+					},
+					3: {
+						{
+							NumaID:   0,
+							Distance: 32,
+						},
+						{
+							NumaID:   1,
+							Distance: 32,
+						},
+						{
+							NumaID:   2,
+							Distance: 11,
+						},
+						{
+							NumaID:   3,
+							Distance: 10,
+						},
+					},
+				},
+			},
+			want: &SiblingNumaInfo{
+				SiblingNumaMap: map[int]sets.Int{
+					0: sets.NewInt(1),
+					1: sets.NewInt(0),
+					2: sets.NewInt(3),
+					3: sets.NewInt(2),
+				},
+				SiblingNumaAvgMBWCapacityMap: map[int]int64{
+					0: 5,
+					1: 5,
+					2: 5,
+					3: 5,
+				},
+				SiblingNumaAvgMBWAllocatableMap: map[int]int64{
+					0: 4,
+					1: 4,
+					2: 4,
+					3: 4,
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		tt := tt