From 45b45bd50111cc385d8ce7d5d34696612f13c236 Mon Sep 17 00:00:00 2001
From: free6om <free6om.c@gmail.com>
Date: Fri, 21 Apr 2023 21:29:42 +0800
Subject: [PATCH] fix: h-scale pvc pending with WaitForFirstConsumer (#2836)

---
 .../transformer_sts_horizontal_scaling.go     | 36 ++++++++++---------
 ...transformer_sts_horizontal_scaling_test.go |  5 ++-
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go
index a9271175b..f2e6cf4f6 100644
--- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go
+++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go
@@ -60,9 +60,6 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 	handleHorizontalScaling := func(vertex *lifecycleVertex) error {
 		stsObj, _ := vertex.oriObj.(*appsv1.StatefulSet)
 		stsProto, _ := vertex.obj.(*appsv1.StatefulSet)
-		if *stsObj.Spec.Replicas == *stsProto.Spec.Replicas {
-			return nil
-		}
 
 		key := client.ObjectKey{
 			Namespace: stsProto.GetNamespace(),
@@ -81,11 +78,13 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 			cluster.Spec.ComponentSpecs)
 		comp := getComponent(components, componentName)
 		if comp == nil {
-			s.ctx.Recorder.Eventf(cluster,
-				corev1.EventTypeWarning,
-				"HorizontalScaleFailed",
-				"component %s not found",
-				componentName)
+			if *stsObj.Spec.Replicas != *stsProto.Spec.Replicas {
+				s.ctx.Recorder.Eventf(cluster,
+					corev1.EventTypeWarning,
+					"HorizontalScaleFailed",
+					"component %s not found",
+					componentName)
+			}
 			return nil
 		}
 		cleanCronJobs := func() error {
@@ -172,13 +171,19 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 				vertex.immutable = true
 				return nil
 			}
-			// check all pvc bound, requeue if not all ready
+			// pvcs are ready, stateful_set.replicas should be updated
+			vertex.immutable = false
+
+			return nil
+		}
+
+		postScaleOut := func() error {
+			// check all pvc bound, wait next reconciliation if not all ready
 			allPVCBounded, err := checkAllPVCBoundIfNeeded()
 			if err != nil {
 				return err
 			}
 			if !allPVCBounded {
-				vertex.immutable = true
 				return nil
 			}
 			// clean backup resources.
@@ -186,12 +191,7 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 			if err := cleanBackupResourcesIfNeeded(); err != nil {
 				return err
 			}
-
-			// pvcs are ready, stateful_set.replicas should be updated
-			vertex.immutable = false
-
 			return nil
-
 		}
 
 		scaleIn := func() error {
@@ -238,6 +238,10 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error {
 				*stsProto.Spec.Replicas)
 		}
 
+		if err = postScaleOut(); err != nil {
+			return err
+		}
+
 		return nil
 	}
 	findPVCsToBeDeleted := func(pvcSnapshot clusterSnapshot) []*corev1.PersistentVolumeClaim {
@@ -521,7 +525,7 @@ func isAllPVCBound(cli types2.ReadonlyClient,
 		pvc := corev1.PersistentVolumeClaim{}
 		// check pvc existence
 		if err := cli.Get(ctx, pvcKey, &pvc); err != nil {
-			return false, err
+			return false, client.IgnoreNotFound(err)
 		}
 		if pvc.Status.Phase != corev1.ClaimBound {
 			return false, nil
diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go
index a2439b062..6b4f0fcd0 100644
--- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go
+++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go
@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/apecloud/kubeblocks/internal/controller/graph"
 	intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil"
@@ -72,8 +73,10 @@ var _ = Describe("sts horizontal scaling test", func() {
 			Expect(intctrlutil.SetOwnership(cluster, pvc2, scheme, dbClusterFinalizerName)).Should(Succeed())
 
 			By("prepare params for transformer")
+			ctx := context.Background()
 			reqCtx := intctrlutil.RequestCtx{
-				Ctx: context.Background(),
+				Ctx: ctx,
+				Log: log.FromContext(ctx).WithValues("transformer", "h-scale"),
 			}
 			ctrl, k8sMock := testutil.SetupK8sMock()
 			defer ctrl.Finish()