feat: add default delete condition to managed dependent resources

Signed-off-by: Attila Mészáros <csviri@gmail.com>

docs/documentation/workflows.md

@@ -312,6 +312,28 @@ containing all the related exceptions.
 The exceptions can be handled
 by [`ErrorStatusHandler`](https://github.com/java-operator-sdk/java-operator-sdk/blob/14620657fcacc8254bb96b4293eded84c20ba685/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ErrorStatusHandler.java)
 
+## Waiting for the actual deletion of Kubernetes Dependent Resources
+
+Let's consider a case when a Kubernetes Dependent Resources (KDR) depends on another resource, on cleanup
+the resources will be deleted in reverse order, thus the KDR will be deleted first.
+However, the workflow implementation currently simply asks the Kubernetes API server to delete the resource. This is,
+however, an asynchronous process, meaning that the deletion might not occur immediately, in particular if the resource
+uses finalizers that block the deletion or if the deletion itself takes some time. From the SDK's perspective, though,
+the deletion has been requested and it moves on to other tasks without waiting for the resource to be actually deleted
+from the server (which might never occur if it uses finalizers which are not removed).
+In situations like these, if your logic depends on resources being actually removed from the cluster before a
+cleanup workflow can proceed correctly, you need to block the workflow progression using a delete post-condition that
+checks that the resource is actually removed or that it, at least, doesn't have any finalizers any longer. JOSDK
+provides such a delete post-condition implementation in the form of
+[`KubernetesResourceDeletedCondition`](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/workflow/KubernetesResourceDeletedCondition.java)
+
+Also, check usage in an [integration test](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/manageddependentdeletecondition/ManagedDependentDefaultDeleteConditionReconciler.java).
+
+In such cases the Kubernetes Dependent Resource should extend `CRUDNoGCKubernetesDependentResource` 
+and NOT `CRUDKubernetesDependentResource` since otherwise the Kubernetes Garbage Collector would delete the resources.
+In other words if a Kubernetes Dependent Resource depends on another dependent resource, it should not implement
+`GargageCollected` interface, otherwise the deletion order won't be guaranteed. 
+
 ## Notes and Caveats
 
 - Delete is almost always called on every resource during the cleanup. However, it might be the case

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/workflow/DefaultManagedWorkflow.java

@@ -85,7 +85,7 @@ public Workflow<P> resolve(KubernetesClient client,
           resolve(spec, client, configuration));
       alreadyResolved.put(node.getName(), node);
       spec.getDependsOn()
-          .forEach(depend -> node.addDependsOnRelation(alreadyResolved.get((String) depend)));
+          .forEach(depend -> node.addDependsOnRelation(alreadyResolved.get(depend)));
     }
 
     final var bottom =

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/workflow/KubernetesResourceDeletedCondition.java

@@ -0,0 +1,27 @@
+package io.javaoperatorsdk.operator.processing.dependent.workflow;
+
+import io.fabric8.kubernetes.api.model.HasMetadata;
+import io.javaoperatorsdk.operator.api.reconciler.Context;
+import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource;
+
+/**
+ * A condition implementation meant to be used as a delete post-condition on Kubernetes dependent
+ * resources to prevent the workflow from proceeding until the associated resource is actually
+ * deleted from the server (or, at least, doesn't have any finalizers anymore). This is needed in
+ * cases where a cleaning process depends on resources being actually removed from the server
+ * because, by default, workflows simply request the deletion but do NOT wait for the resources to
+ * be actually deleted.
+ */
+public class KubernetesResourceDeletedCondition implements Condition<HasMetadata, HasMetadata> {
+
+  @Override
+  public boolean isMet(DependentResource<HasMetadata, HasMetadata> dependentResource,
+      HasMetadata primary, Context<HasMetadata> context) {
+    var optionalResource = dependentResource.getSecondaryResource(primary, context);
+    if (optionalResource.isEmpty()) {
+      return true;
+    } else {
+      return optionalResource.orElseThrow().getMetadata().getFinalizers().isEmpty();
+    }
+  }
+}

operator-framework/src/test/java/io/javaoperatorsdk/operator/ManagedDependentDeleteConditionIT.java

@@ -0,0 +1,71 @@
+package io.javaoperatorsdk.operator;
+
+import java.time.Duration;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.fabric8.kubernetes.api.model.Secret;
+import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension;
+import io.javaoperatorsdk.operator.sample.manageddependentdeletecondition.ManagedDependentDefaultDeleteConditionCustomResource;
+import io.javaoperatorsdk.operator.sample.manageddependentdeletecondition.ManagedDependentDefaultDeleteConditionReconciler;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+public class ManagedDependentDeleteConditionIT {
+
+  public static final String RESOURCE_NAME = "test1";
+  public static final String CUSTOM_FINALIZER = "test/customfinalizer";
+
+  @RegisterExtension
+  LocallyRunOperatorExtension extension =
+      LocallyRunOperatorExtension.builder()
+          .withConfigurationService(o -> o.withDefaultNonSSAResource(Set.of()))
+          .withReconciler(new ManagedDependentDefaultDeleteConditionReconciler()).build();
+
+
+  @Test
+  void resourceNotDeletedUntilDependentDeleted() {
+    var resource = new ManagedDependentDefaultDeleteConditionCustomResource();
+    resource.setMetadata(new ObjectMetaBuilder()
+        .withName(RESOURCE_NAME)
+        .build());
+    resource = extension.create(resource);
+
+    await().timeout(Duration.ofSeconds(300)).untilAsserted(() -> {
+      var cm = extension.get(ConfigMap.class, RESOURCE_NAME);
+      var sec = extension.get(Secret.class, RESOURCE_NAME);
+      assertThat(cm).isNotNull();
+      assertThat(sec).isNotNull();
+    });
+
+    var secret = extension.get(Secret.class, RESOURCE_NAME);
+    secret.getMetadata().getFinalizers().add(CUSTOM_FINALIZER);
+    secret = extension.replace(secret);
+
+    extension.delete(resource);
+
+    // both resources are present until the finalizer removed
+    await().pollDelay(Duration.ofMillis(250)).untilAsserted(() -> {
+      var cm = extension.get(ConfigMap.class, RESOURCE_NAME);
+      var sec = extension.get(Secret.class, RESOURCE_NAME);
+      assertThat(cm).isNotNull();
+      assertThat(sec).isNotNull();
+    });
+
+    secret.getMetadata().getFinalizers().clear();
+    extension.replace(secret);
+
+    await().untilAsserted(() -> {
+      var cm = extension.get(ConfigMap.class, RESOURCE_NAME);
+      var sec = extension.get(Secret.class, RESOURCE_NAME);
+      assertThat(cm).isNull();
+      assertThat(sec).isNull();
+    });
+  }
+
+}

operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/manageddependentdeletecondition/ConfigMapDependent.java

@@ -0,0 +1,29 @@
+package io.javaoperatorsdk.operator.sample.manageddependentdeletecondition;
+
+import java.util.Map;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
+import io.javaoperatorsdk.operator.api.reconciler.Context;
+import io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDNoGCKubernetesDependentResource;
+
+public class ConfigMapDependent extends
+    CRUDNoGCKubernetesDependentResource<ConfigMap, ManagedDependentDefaultDeleteConditionCustomResource> {
+
+  public ConfigMapDependent() {
+    super(ConfigMap.class);
+  }
+
+  @Override
+  protected ConfigMap desired(ManagedDependentDefaultDeleteConditionCustomResource primary,
+      Context<ManagedDependentDefaultDeleteConditionCustomResource> context) {
+
+    return new ConfigMapBuilder()
+        .withNewMetadata()
+        .withName(primary.getMetadata().getName())
+        .withNamespace(primary.getMetadata().getNamespace())
+        .endMetadata()
+        .withData(Map.of("key", "val"))
+        .build();
+  }
+}

operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/manageddependentdeletecondition/ManagedDependentDefaultDeleteConditionCustomResource.java

@@ -0,0 +1,15 @@
+package io.javaoperatorsdk.operator.sample.manageddependentdeletecondition;
+
+import io.fabric8.kubernetes.api.model.Namespaced;
+import io.fabric8.kubernetes.client.CustomResource;
+import io.fabric8.kubernetes.model.annotation.Group;
+import io.fabric8.kubernetes.model.annotation.ShortNames;
+import io.fabric8.kubernetes.model.annotation.Version;
+
+@Group("sample.javaoperatorsdk")
+@Version("v1")
+@ShortNames("mdcc")
+public class ManagedDependentDefaultDeleteConditionCustomResource
+    extends CustomResource<Void, Void>
+    implements Namespaced {
+}

operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/manageddependentdeletecondition/ManagedDependentDefaultDeleteConditionReconciler.java

@@ -0,0 +1,31 @@
+package io.javaoperatorsdk.operator.sample.manageddependentdeletecondition;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.javaoperatorsdk.operator.api.reconciler.*;
+import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent;
+import io.javaoperatorsdk.operator.processing.dependent.workflow.KubernetesResourceDeletedCondition;
+
+@ControllerConfiguration(dependents = {
+    @Dependent(name = "ConfigMap", type = ConfigMapDependent.class),
+    @Dependent(type = SecretDependent.class, dependsOn = "ConfigMap",
+        deletePostcondition = KubernetesResourceDeletedCondition.class)
+})
+public class ManagedDependentDefaultDeleteConditionReconciler
+    implements Reconciler<ManagedDependentDefaultDeleteConditionCustomResource> {
+
+  private static final Logger log =
+      LoggerFactory.getLogger(ManagedDependentDefaultDeleteConditionReconciler.class);
+
+  @Override
+  public UpdateControl<ManagedDependentDefaultDeleteConditionCustomResource> reconcile(
+      ManagedDependentDefaultDeleteConditionCustomResource resource,
+      Context<ManagedDependentDefaultDeleteConditionCustomResource> context) {
+
+    log.debug("Reconciled: {}", resource);
+
+    return UpdateControl.noUpdate();
+  }
+
+}

operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/manageddependentdeletecondition/SecretDependent.java

@@ -0,0 +1,33 @@
+package io.javaoperatorsdk.operator.sample.manageddependentdeletecondition;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Map;
+
+import io.fabric8.kubernetes.api.model.Secret;
+import io.fabric8.kubernetes.api.model.SecretBuilder;
+import io.javaoperatorsdk.operator.api.reconciler.Context;
+import io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDNoGCKubernetesDependentResource;
+
+public class SecretDependent
+    extends
+    CRUDNoGCKubernetesDependentResource<Secret, ManagedDependentDefaultDeleteConditionCustomResource> {
+
+  public SecretDependent() {
+    super(Secret.class);
+  }
+
+  @Override
+  protected Secret desired(ManagedDependentDefaultDeleteConditionCustomResource primary,
+      Context<ManagedDependentDefaultDeleteConditionCustomResource> context) {
+
+    return new SecretBuilder()
+        .withNewMetadata()
+        .withName(primary.getMetadata().getName())
+        .withNamespace(primary.getMetadata().getNamespace())
+        .endMetadata()
+        .withData(Map.of("key",
+            new String(Base64.getEncoder().encode("val".getBytes(StandardCharsets.UTF_16)))))
+        .build();
+  }
+}