fix: h-scale pvc pending with WaitForFirstConsumer (#2836)

apecloud · Apr 21, 2023 · 45b45bd · 45b45bd
1 parent 919e4c7
commit 45b45bd
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 17 deletions.
diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go
@@ -60,9 +60,6 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 	handleHorizontalScaling := func(vertex *lifecycleVertex) error {
 		stsObj, _ := vertex.oriObj.(*appsv1.StatefulSet)
 		stsProto, _ := vertex.obj.(*appsv1.StatefulSet)
-		if *stsObj.Spec.Replicas == *stsProto.Spec.Replicas {
-			return nil
-		}
 
 		key := client.ObjectKey{
 			Namespace: stsProto.GetNamespace(),
@@ -81,11 +78,13 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 			cluster.Spec.ComponentSpecs)
 		comp := getComponent(components, componentName)
 		if comp == nil {
-			s.ctx.Recorder.Eventf(cluster,
-				corev1.EventTypeWarning,
-				"HorizontalScaleFailed",
-				"component %s not found",
-				componentName)
+			if *stsObj.Spec.Replicas != *stsProto.Spec.Replicas {
+				s.ctx.Recorder.Eventf(cluster,
+					corev1.EventTypeWarning,
+					"HorizontalScaleFailed",
+					"component %s not found",
+					componentName)
+			}
 			return nil
 		}
 		cleanCronJobs := func() error {
@@ -172,26 +171,27 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 				vertex.immutable = true
 				return nil
 			}
-			// check all pvc bound, requeue if not all ready
+			// pvcs are ready, stateful_set.replicas should be updated
+			vertex.immutable = false
+
+			return nil
+		}
+
+		postScaleOut := func() error {
+			// check all pvc bound, wait next reconciliation if not all ready
 			allPVCBounded, err := checkAllPVCBoundIfNeeded()
 			if err != nil {
 				return err
 			}
 			if !allPVCBounded {
-				vertex.immutable = true
 				return nil
 			}
 			// clean backup resources.
 			// there will not be any backup resources other than scale out.
 			if err := cleanBackupResourcesIfNeeded(); err != nil {
 				return err
 			}
-
-			// pvcs are ready, stateful_set.replicas should be updated
-			vertex.immutable = false
-
 			return nil
-
 		}
 
 		scaleIn := func() error {
@@ -238,6 +238,10 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 				*stsProto.Spec.Replicas)
 		}
 
+		if err = postScaleOut(); err != nil {
+			return err
+		}
+
 		return nil
 	}
 	findPVCsToBeDeleted := func(pvcSnapshot clusterSnapshot) []*corev1.PersistentVolumeClaim {
@@ -521,7 +525,7 @@ func isAllPVCBound(cli types2.ReadonlyClient,
 		pvc := corev1.PersistentVolumeClaim{}
 		// check pvc existence
 		if err := cli.Get(ctx, pvcKey, &pvc); err != nil {
-			return false, err
+			return false, client.IgnoreNotFound(err)
 		}
 		if pvc.Status.Phase != corev1.ClaimBound {
 			return false, nil

diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go
@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/apecloud/kubeblocks/internal/controller/graph"
 	intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil"
@@ -72,8 +73,10 @@ var _ = Describe("sts horizontal scaling test", func() {
 			Expect(intctrlutil.SetOwnership(cluster, pvc2, scheme, dbClusterFinalizerName)).Should(Succeed())
 
 			By("prepare params for transformer")
+			ctx := context.Background()
 			reqCtx := intctrlutil.RequestCtx{
-				Ctx: context.Background(),
+				Ctx: ctx,
+				Log: log.FromContext(ctx).WithValues("transformer", "h-scale"),
 			}
 			ctrl, k8sMock := testutil.SetupK8sMock()
 			defer ctrl.Finish()