diff --git a/charts/spiderpool/templates/deployment.yaml b/charts/spiderpool/templates/deployment.yaml index 777ec9a3a0..de517a0f2d 100644 --- a/charts/spiderpool/templates/deployment.yaml +++ b/charts/spiderpool/templates/deployment.yaml @@ -161,10 +161,12 @@ spec: value: {{ .Values.spiderpoolController.httpPort | quote }} - name: SPIDERPOOL_GC_IP_ENABLED value: {{ .Values.ipam.gc.enabled | quote }} - - name: SPIDERPOOL_GC_TERMINATING_POD_IP_ENABLED - value: {{ .Values.ipam.gc.GcDeletingTimeOutPod.enabled | quote }} + - name: SPIDERPOOL_GC_TERMINATING_NODE_READY_POD_IP_ENABLED + value: {{ .Values.ipam.gc.enableGcDeletingTimeOutPodWithNodeReady | quote }} + - name: SPIDERPOOL_GC_TERMINATING_NODE_NOT_READY_POD_IP_ENABLED + value: {{ .Values.ipam.gc.enableGcDeletingTimeOutPodWithNodeNotReady | quote }} - name: SPIDERPOOL_GC_ADDITIONAL_GRACE_DELAY - value: {{ .Values.ipam.gc.GcDeletingTimeOutPod.delay | quote }} + value: {{ .Values.ipam.gc.gcDeletingTimeOutPodDelay | quote }} - name: SPIDERPOOL_GC_DEFAULT_INTERVAL_DURATION value: {{ .Values.ipam.gc.gcAll.intervalInSecond | quote }} - name: SPIDERPOOL_MULTUS_CONFIG_ENABLED diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml index 142c9d0672..939245d241 100644 --- a/charts/spiderpool/values.yaml +++ b/charts/spiderpool/values.yaml @@ -67,12 +67,14 @@ ipam: ## @param ipam.gc.gcAll.intervalInSecond the gc all interval duration intervalInSecond: 600 - GcDeletingTimeOutPod: - ## @param ipam.gc.GcDeletingTimeOutPod.enabled enable retrieve IP for the pod who times out of deleting graceful period - enabled: true + ## @param ipam.gc.enableGcDeletingTimeOutPodWithNodeReady enable reclaim IP for the pod who times out of deleting graceful period with its node ready + enableGcDeletingTimeOutPodWithNodeReady: true - ## @param ipam.gc.GcDeletingTimeOutPod.delay the gc delay seconds after the pod times out of deleting graceful period - delay: 0 + ## @param ipam.gc.enableGcDeletingTimeOutPodWithNodeNotReady enable reclaim IP for the pod who times out of deleting graceful period with its node not ready + enableGcDeletingTimeOutPodWithNodeNotReady: true + + ## @param ipam.gc.gcDeletingTimeOutPodDelay the gc delay seconds after the pod times out of deleting graceful period + gcDeletingTimeOutPodDelay: 0 ## @section grafanaDashboard parameters ## diff --git a/cmd/spiderpool-controller/cmd/config.go b/cmd/spiderpool-controller/cmd/config.go index b48c91fbc1..ce4b02a54f 100644 --- a/cmd/spiderpool-controller/cmd/config.go +++ b/cmd/spiderpool-controller/cmd/config.go @@ -64,7 +64,11 @@ var envInfo = []envConf{ {"SPIDERPOOL_PYROSCOPE_PUSH_SERVER_ADDRESS", "", false, &controllerContext.Cfg.PyroscopeAddress, nil, nil}, {"SPIDERPOOL_GC_IP_ENABLED", "true", true, nil, &gcIPConfig.EnableGCIP, nil}, - {"SPIDERPOOL_GC_TERMINATING_POD_IP_ENABLED", "true", true, nil, &gcIPConfig.EnableGCForTerminatingPod, nil}, + //{"SPIDERPOOL_GC_TERMINATING_POD_IP_ENABLED", "true", true, nil, &gcIPConfig.EnableGCForTerminatingPod, nil}, + + {"SPIDERPOOL_GC_TERMINATING_NODE_READY_POD_IP_ENABLED", "true", true, nil, &gcIPConfig.EnableGCForTerminatingPodWithNodeReady, nil}, + {"SPIDERPOOL_GC_TERMINATING_NODE_NOT_READY_POD_IP_ENABLED", "true", true, nil, &gcIPConfig.EnableGCForTerminatingPodWithNodeNotReady, nil}, + {"SPIDERPOOL_GC_IP_WORKER_NUM", "3", true, nil, nil, &gcIPConfig.ReleaseIPWorkerNum}, {"SPIDERPOOL_GC_CHANNEL_BUFFER", "5000", true, nil, nil, &gcIPConfig.GCIPChannelBuffer}, {"SPIDERPOOL_GC_MAX_PODENTRY_DB_CAP", "100000", true, nil, nil, &gcIPConfig.MaxPodEntryDatabaseCap}, diff --git a/cmd/spiderpool-controller/cmd/daemon.go b/cmd/spiderpool-controller/cmd/daemon.go index 90d4bef2dc..38393b85e2 100644 --- a/cmd/spiderpool-controller/cmd/daemon.go +++ b/cmd/spiderpool-controller/cmd/daemon.go @@ -380,6 +380,7 @@ func initGCManager(ctx context.Context) { controllerContext.PodManager, controllerContext.StsManager, controllerContext.KubevirtManager, + controllerContext.NodeManager, controllerContext.Leader, ) if nil != err { diff --git a/pkg/gcmanager/gc_manager.go b/pkg/gcmanager/gc_manager.go index d896638337..72d7dc55a5 100644 --- a/pkg/gcmanager/gc_manager.go +++ b/pkg/gcmanager/gc_manager.go @@ -17,16 +17,18 @@ import ( "github.com/spidernet-io/spiderpool/pkg/kubevirtmanager" "github.com/spidernet-io/spiderpool/pkg/limiter" "github.com/spidernet-io/spiderpool/pkg/logutils" + "github.com/spidernet-io/spiderpool/pkg/nodemanager" "github.com/spidernet-io/spiderpool/pkg/podmanager" "github.com/spidernet-io/spiderpool/pkg/statefulsetmanager" "github.com/spidernet-io/spiderpool/pkg/workloadendpointmanager" ) type GarbageCollectionConfig struct { - EnableGCIP bool - EnableGCForTerminatingPod bool - EnableStatefulSet bool - EnableKubevirtStaticIP bool + EnableGCIP bool + EnableGCForTerminatingPodWithNodeReady bool + EnableGCForTerminatingPodWithNodeNotReady bool + EnableStatefulSet bool + EnableKubevirtStaticIP bool ReleaseIPWorkerNum int GCIPChannelBuffer int @@ -69,6 +71,7 @@ type SpiderGC struct { podMgr podmanager.PodManager stsMgr statefulsetmanager.StatefulSetManager kubevirtMgr kubevirtmanager.KubevirtManager + nodeMgr nodemanager.NodeManager leader election.SpiderLeaseElector informerFactory informers.SharedInformerFactory @@ -81,6 +84,7 @@ func NewGCManager(clientSet *kubernetes.Clientset, config *GarbageCollectionConf podManager podmanager.PodManager, stsManager statefulsetmanager.StatefulSetManager, kubevirtMgr kubevirtmanager.KubevirtManager, + nodeMgr nodemanager.NodeManager, spiderControllerLeader election.SpiderLeaseElector) (GCManager, error) { if clientSet == nil { return nil, fmt.Errorf("k8s ClientSet must be specified") @@ -121,6 +125,7 @@ func NewGCManager(clientSet *kubernetes.Clientset, config *GarbageCollectionConf podMgr: podManager, stsMgr: stsManager, kubevirtMgr: kubevirtMgr, + nodeMgr: nodeMgr, leader: spiderControllerLeader, gcLimiter: limiter.NewLimiter(limiter.LimiterConfig{}), diff --git a/pkg/gcmanager/pod_cache.go b/pkg/gcmanager/pod_cache.go index 88a60b3684..5bc5678644 100644 --- a/pkg/gcmanager/pod_cache.go +++ b/pkg/gcmanager/pod_cache.go @@ -17,6 +17,7 @@ import ( "github.com/spidernet-io/spiderpool/pkg/constant" "github.com/spidernet-io/spiderpool/pkg/lock" "github.com/spidernet-io/spiderpool/pkg/logutils" + "github.com/spidernet-io/spiderpool/pkg/nodemanager" "github.com/spidernet-io/spiderpool/pkg/types" ) @@ -224,9 +225,19 @@ func (s *SpiderGC) buildPodEntry(oldPod, currentPod *corev1.Pod, deleted bool) ( } if isBuildTerminatingPodEntry { - // disable for gc terminating pod - if !s.gcConfig.EnableGCForTerminatingPod { - logger.Sugar().Debugf("IP gc already turn off 'EnableGCForTerminatingPod' configuration, disacrd tracing pod '%s/%s'", currentPod.Namespace, currentPod.Name) + // check terminating Pod corresponding Node status + node, err := s.nodeMgr.GetNodeByName(ctx, currentPod.Spec.NodeName, constant.UseCache) + if nil != err { + return nil, fmt.Errorf("failed to get terminating Pod '%s/%s' corredponing Node '%s', error: %v", currentPod.Namespace, currentPod.Name, currentPod.Spec.NodeName) + } + // disable for gc terminating pod with Node Ready + if nodemanager.IsNodeReady(node) && !s.gcConfig.EnableGCForTerminatingPodWithNodeReady { + logger.Sugar().Debugf("IP GC already turn off 'EnableGCForTerminatingPodWithNodeReady' configuration, disacrd tracing pod '%s/%s'", currentPod.Namespace, currentPod.Name) + return nil, nil + } + // disable for gc terminating pod with Node NotReady + if !nodemanager.IsNodeReady(node) && !s.gcConfig.EnableGCForTerminatingPodWithNodeNotReady { + logger.Sugar().Debugf("IP GC already turn off 'EnableGCForTerminatingPodWithNodeNotReady' configuration, disacrd tracing pod '%s/%s'", currentPod.Namespace, currentPod.Name) return nil, nil } diff --git a/pkg/nodemanager/utils.go b/pkg/nodemanager/utils.go new file mode 100644 index 0000000000..e403fce9a8 --- /dev/null +++ b/pkg/nodemanager/utils.go @@ -0,0 +1,18 @@ +// Copyright 2023 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package nodemanager + +import corev1 "k8s.io/api/core/v1" + +func IsNodeReady(node *corev1.Node) bool { + var readyCondition corev1.NodeCondition + for _, tmpCondition := range node.Status.Conditions { + if tmpCondition.Type == corev1.NodeReady { + readyCondition = tmpCondition + break + } + } + + return readyCondition.Status == corev1.ConditionTrue +} diff --git a/pkg/nodemanager/utils_test.go b/pkg/nodemanager/utils_test.go new file mode 100644 index 0000000000..03bc1bad13 --- /dev/null +++ b/pkg/nodemanager/utils_test.go @@ -0,0 +1,48 @@ +// Copyright 2023 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package nodemanager + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("NodeManager utils", Label("node_manager_utils_test"), func() { + Describe("IsNodeReady", func() { + var node *corev1.Node + BeforeEach(func() { + node = &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + }, + Spec: corev1.NodeSpec{}, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeMemoryPressure, + Status: corev1.ConditionFalse, + }, + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } + }) + + It("Node is ready", func() { + isNodeReady := IsNodeReady(node) + Expect(isNodeReady).To(BeTrue()) + }) + + It("Node is not ready", func() { + node.Status.Conditions[1].Status = corev1.ConditionUnknown + isNodeReady := IsNodeReady(node) + Expect(isNodeReady).To(BeFalse()) + }) + }) +})