Add missing requeue to Datacenter decommission (#322)

* Add requeue if we still have pods although decommission has succeeded * CHANGELOG
k8ssandra · Apr 21, 2022 · addd528 · addd528
1 parent b9d08bf
commit addd528
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 16 deletions.
diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml
@@ -133,7 +133,7 @@ jobs:
         - "3.11.7"
         - "3.11.11"
         - "4.0.0"
-        - "4.0.1"
+        - "4.0.3"
         include:
           - version: 3.11.7
             serverImage: k8ssandra/cass-management-api:3.11.7-v0.1.24 # k8ssandra 1.1

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
 * [CHANGE] [#183](https://github.com/k8ssandra/cass-operator/issues/183) Move from PodDisruptionBudget v1beta1 to v1 (min. Kubernetes version 1.21)
 * [ENHANCEMENT] [#313](https://github.com/k8ssandra/cass-operator/issues/313) Add support for Cassandra 4.x.x versions instead of only 4.0.x
 * [ENHANCEMENT] [#292](https://github.com/k8ssandra/cass-operator/issues/292) Update to Go 1.17 with updates to dependencies: Kube 1.23.4 and controller-runtime 0.11.1 
+* [BUGFIX] [#322](https://github.com/k8ssandra/cass-operator/pull/322) Add missing requeue if decommissioned pods haven't been removed y et
 
 ## v1.10.3
 

diff --git a/pkg/reconciliation/reconcile_datacenter.go b/pkg/reconciliation/reconcile_datacenter.go
@@ -70,6 +70,8 @@ func (rc *ReconciliationContext) ProcessDeletion() result.ReconcileResult {
 				// Exiting to let other parts of the process take care of the decommission
 				return result.Continue()
 			}
+			// How could we have pods if we've decommissioned everything?
+			return result.RequeueSoon(5)
 		}
 	}
 

diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go
@@ -2132,27 +2132,32 @@ func (rc *ReconciliationContext) cleanupAfterScaling() result.ReconcileResult {
 	}
 
 	if task != nil {
-		if task.Status.CompletionTime != nil {
-			// Job was completed, remove it from followed task
-			dc := rc.Datacenter
-			dcPatch := client.MergeFrom(dc.DeepCopy())
+		return rc.activeTaskCompleted(task)
+	}
 
-			rc.Datacenter.Status.RemoveTrackedTask(task.ObjectMeta)
+	// Create the cleanup task
+	err = rc.createTask("cleanup")
+	if err != nil {
+		return result.Error(err)
+	}
 
-			if err := rc.Client.Status().Patch(rc.Ctx, dc, dcPatch); err != nil {
-				return result.Error(err)
-			}
+	return result.RequeueSoon(10)
+}
 
-			return result.Continue()
-		}
-	} else {
-		// Create the cleanup task
-		err := rc.createTask("cleanup")
-		if err != nil {
+func (rc *ReconciliationContext) activeTaskCompleted(task *taskapi.CassandraTask) result.ReconcileResult {
+	if task.Status.CompletionTime != nil {
+		// Job was completed, remove it from followed task
+		dc := rc.Datacenter
+		dcPatch := client.MergeFrom(dc.DeepCopy())
+
+		rc.Datacenter.Status.RemoveTrackedTask(task.ObjectMeta)
+
+		if err := rc.Client.Status().Patch(rc.Ctx, dc, dcPatch); err != nil {
 			return result.Error(err)
 		}
-	}
 
+		return result.Continue()
+	}
 	return result.RequeueSoon(10)
 }