Skip to content

Commit

Permalink
add namespace filter to nodeutilization
Browse files Browse the repository at this point in the history
  • Loading branch information
knelasevero committed Oct 25, 2022
1 parent 7777d5a commit a646d62
Show file tree
Hide file tree
Showing 8 changed files with 180 additions and 3 deletions.
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ actual usage metrics. Implementing metrics-based descheduling is currently TODO
|`thresholdPriority`|int (see [priority filtering](#priority-filtering))|
|`thresholdPriorityClassName`|string (see [priority filtering](#priority-filtering))|
|`nodeFit`|bool (see [node fit filtering](#node-fit-filtering))|
|`evictionNamespaces`|(see [namespace filtering](#namespace-filtering))|

**Example:**

Expand Down Expand Up @@ -315,6 +316,7 @@ actual usage metrics. Implementing metrics-based descheduling is currently TODO
|`thresholdPriority`|int (see [priority filtering](#priority-filtering))|
|`thresholdPriorityClassName`|string (see [priority filtering](#priority-filtering))|
|`nodeFit`|bool (see [node fit filtering](#node-fit-filtering))|
|`evictionNamespaces`|(see [namespace filtering](#namespace-filtering))|

**Example:**

Expand Down Expand Up @@ -613,6 +615,7 @@ The following strategies accept a `namespaces` parameter which allows to specify
* `RemoveDuplicates`
* `RemovePodsViolatingTopologySpreadConstraint`
* `RemoveFailedPods`
* `LowNodeUtilization` and `HighNodeUtilization` (Only filtered right before eviction, parameter named `evictionNamespaces`)

For example:

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ func (h *HighNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fr

evictPodsFromSourceNodes(
ctx,
h.args.EvictableNamespaces,
sourceNodes,
highNodes,
h.handle.Evictor(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fra

evictPodsFromSourceNodes(
ctx,
l.args.EvictableNamespaces,
sourceNodes,
lowNodes,
l.handle.Evictor(),
Expand Down
126 changes: 126 additions & 0 deletions pkg/framework/plugins/nodeutilization/lownodeutilization_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ func TestLowNodeUtilization(t *testing.T) {
pods []*v1.Pod
expectedPodsEvicted uint
evictedPods []string
evictableNamespaces *api.Namespaces
}{
{
name: "no evictable pods",
Expand Down Expand Up @@ -155,6 +156,130 @@ func TestLowNodeUtilization(t *testing.T) {
},
expectedPodsEvicted: 4,
},
{
name: "without priorities, but excluding namespaces",
thresholds: api.ResourceThresholds{
v1.ResourceCPU: 30,
v1.ResourcePods: 30,
},
targetThresholds: api.ResourceThresholds{
v1.ResourceCPU: 50,
v1.ResourcePods: 50,
},
nodes: []*v1.Node{
test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil),
test.BuildTestNode(n2NodeName, 4000, 3000, 10, nil),
test.BuildTestNode(n3NodeName, 4000, 3000, 10, test.SetNodeUnschedulable),
},
pods: []*v1.Pod{
test.BuildTestPod("p1", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace1"
}),
test.BuildTestPod("p2", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace1"
}),
test.BuildTestPod("p3", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace1"
}),
test.BuildTestPod("p4", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace1"
}),
test.BuildTestPod("p5", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace1"
}),
// These won't be evicted.
test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef),
test.BuildTestPod("p7", 400, 0, n1NodeName, func(pod *v1.Pod) {
// A pod with local storage.
test.SetNormalOwnerRef(pod)
pod.Spec.Volumes = []v1.Volume{
{
Name: "sample",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "somePath"},
EmptyDir: &v1.EmptyDirVolumeSource{
SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)},
},
},
}
// A Mirror Pod.
pod.Annotations = test.GetMirrorPodAnnotation()
}),
test.BuildTestPod("p8", 400, 0, n1NodeName, func(pod *v1.Pod) {
// A Critical Pod.
pod.Namespace = "kube-system"
priority := utils.SystemCriticalPriority
pod.Spec.Priority = &priority
}),
test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
},
evictableNamespaces: &api.Namespaces{
Exclude: []string{
"namespace1",
},
},
expectedPodsEvicted: 0,
},
{
name: "without priorities, but include only default namespace",
thresholds: api.ResourceThresholds{
v1.ResourceCPU: 30,
v1.ResourcePods: 30,
},
targetThresholds: api.ResourceThresholds{
v1.ResourceCPU: 50,
v1.ResourcePods: 50,
},
nodes: []*v1.Node{
test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil),
test.BuildTestNode(n2NodeName, 4000, 3000, 10, nil),
test.BuildTestNode(n3NodeName, 4000, 3000, 10, test.SetNodeUnschedulable),
},
pods: []*v1.Pod{
test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef),
test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef),
test.BuildTestPod("p3", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace3"
}),
test.BuildTestPod("p4", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace4"
}),
test.BuildTestPod("p5", 400, 0, n1NodeName, func(pod *v1.Pod) {
pod.Namespace = "namespace5"
}),
// These won't be evicted.
test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef),
test.BuildTestPod("p7", 400, 0, n1NodeName, func(pod *v1.Pod) {
// A pod with local storage.
test.SetNormalOwnerRef(pod)
pod.Spec.Volumes = []v1.Volume{
{
Name: "sample",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "somePath"},
EmptyDir: &v1.EmptyDirVolumeSource{
SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)},
},
},
}
// A Mirror Pod.
pod.Annotations = test.GetMirrorPodAnnotation()
}),
test.BuildTestPod("p8", 400, 0, n1NodeName, func(pod *v1.Pod) {
// A Critical Pod.
pod.Namespace = "kube-system"
priority := utils.SystemCriticalPriority
pod.Spec.Priority = &priority
}),
test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
},
evictableNamespaces: &api.Namespaces{
Include: []string{
"default",
},
},
expectedPodsEvicted: 2,
},
{
name: "without priorities stop when cpu capacity is depleted",
thresholds: api.ResourceThresholds{
Expand Down Expand Up @@ -794,6 +919,7 @@ func TestLowNodeUtilization(t *testing.T) {
Thresholds: test.thresholds,
TargetThresholds: test.targetThresholds,
UseDeviationThresholds: test.useDeviationThresholds,
EvictableNamespaces: test.evictableNamespaces,
},
handle)
if err != nil {
Expand Down
23 changes: 21 additions & 2 deletions pkg/framework/plugins/nodeutilization/nodeutilization.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
"sigs.k8s.io/descheduler/pkg/descheduler/node"
Expand Down Expand Up @@ -212,6 +213,7 @@ func classifyNodes(
// TODO: @ravig Break this function into smaller functions.
func evictPodsFromSourceNodes(
ctx context.Context,
evictionNamespaces *api.Namespaces,
sourceNodes, destinationNodes []NodeInfo,
podEvictor framework.Evictor,
podFilter func(pod *v1.Pod) bool,
Expand Down Expand Up @@ -265,13 +267,14 @@ func evictPodsFromSourceNodes(
klog.V(1).InfoS("Evicting pods based on priority, if they have same priority, they'll be evicted based on QoS tiers")
// sort the evictable Pods based on priority. This also sorts them based on QoS. If there are multiple pods with same priority, they are sorted based on QoS tiers.
podutil.SortPodsBasedOnPriorityLowToHigh(removablePods)
evictPods(ctx, removablePods, node, totalAvailableUsage, taintsOfDestinationNodes, podEvictor, continueEviction)
evictPods(ctx, evictionNamespaces, removablePods, node, totalAvailableUsage, taintsOfDestinationNodes, podEvictor, continueEviction)

}
}

func evictPods(
ctx context.Context,
evictionNamespaces *api.Namespaces,
inputPods []*v1.Pod,
nodeInfo NodeInfo,
totalAvailableUsage map[v1.ResourceName]*resource.Quantity,
Expand All @@ -280,14 +283,30 @@ func evictPods(
continueEviction continueEvictionCond,
) {

var includedNamespaces, excludedNamespaces sets.String
if evictionNamespaces != nil {
includedNamespaces = sets.NewString(evictionNamespaces.Include...)
excludedNamespaces = sets.NewString(evictionNamespaces.Exclude...)
}

if continueEviction(nodeInfo, totalAvailableUsage) {
for _, pod := range inputPods {
if !utils.PodToleratesTaints(pod, taintsOfLowNodes) {
klog.V(3).InfoS("Skipping eviction for pod, doesn't tolerate node taint", "pod", klog.KObj(pod))
continue
}

if podEvictor.PreEvictionFilter(pod) {
preEvictionFilterWithOptions, err := podutil.NewOptions().
WithFilter(podEvictor.PreEvictionFilter).
WithNamespaces(includedNamespaces).
WithoutNamespaces(excludedNamespaces).
BuildFilterFunc()
if err != nil {
klog.V(1).ErrorS(err, "could not build preEvictionFilter with namespace inclusion/exclusion")
continue
}

if preEvictionFilterWithOptions(pod) {
if podEvictor.Evict(ctx, pod, evictions.EvictOptions{}) {
klog.V(3).InfoS("Evicted pods", "pod", klog.KObj(pod))

Expand Down
8 changes: 8 additions & 0 deletions pkg/framework/plugins/nodeutilization/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ type LowNodeUtilizationArgs struct {
Thresholds api.ResourceThresholds
TargetThresholds api.ResourceThresholds
NumberOfNodes int
// Naming this one differently since namespaces are still
// considered while considering resoures used by pods
// but then filtered out before eviction
EvictableNamespaces *api.Namespaces
}

// +k8s:deepcopy-gen=true
Expand All @@ -38,4 +42,8 @@ type HighNodeUtilizationArgs struct {

Thresholds api.ResourceThresholds
NumberOfNodes int
// Naming this one differently since namespaces are still
// considered while considering resoures used by pods
// but then filtered out before eviction
EvictableNamespaces *api.Namespaces
}
11 changes: 10 additions & 1 deletion pkg/framework/plugins/nodeutilization/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,16 @@ import (
)

func ValidateHighNodeUtilizationArgs(args *HighNodeUtilizationArgs) error {
return validateThresholds(args.Thresholds)
// only exclude can be set, or not at all
if args.EvictableNamespaces != nil && len(args.EvictableNamespaces.Include) > 0 {
return fmt.Errorf("only Exclude namespaces can be set, inclusion is not supported")
}
err := validateThresholds(args.Thresholds)
if err != nil {
return err
}

return nil
}

func ValidateLowNodeUtilizationArgs(args *LowNodeUtilizationArgs) error {
Expand Down
10 changes: 10 additions & 0 deletions pkg/framework/plugins/nodeutilization/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit a646d62

Please sign in to comment.