From 1ea9c9d6a41e3b1f5f687d66ebe319ebcbc34c43 Mon Sep 17 00:00:00 2001 From: apedriza Date: Thu, 16 Jan 2025 15:45:03 +0100 Subject: [PATCH] Remove JoinTokenRequest resource if cluster it belongs has been removed (#884) Signed-off-by: Adrian Pedriza Co-authored-by: Adrian Pedriza --- .../jointokenrequest_controller.go | 18 ++++++++- .../k0smotroncluster_controller.go | 40 ++++++++++++++++++- inttest/jointoken/jointoken_test.go | 21 ++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/internal/controller/k0smotron.io/jointokenrequest_controller.go b/internal/controller/k0smotron.io/jointokenrequest_controller.go index cf87f714c..924ffc7f5 100644 --- a/internal/controller/k0smotron.io/jointokenrequest_controller.go +++ b/internal/controller/k0smotron.io/jointokenrequest_controller.go @@ -74,7 +74,23 @@ func (r *JoinTokenRequestReconciler) Reconcile(ctx context.Context, req ctrl.Req r.updateStatus(ctx, jtr, "Failed getting cluster") return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err } - jtr.Status.ClusterUID = cluster.GetUID() + clusterUID := cluster.GetUID() + jtr.Status.ClusterUID = clusterUID + + if !controllerutil.ContainsFinalizer(&cluster, clusterFinalizer) { + cluster.Finalizers = append(cluster.Finalizers, clusterFinalizer) + } + err = r.Update(ctx, &cluster) + if err != nil { + logger.Error(err, "unable to add finalizer to cluster for JoinTokenRequest resource removal") + } + + jtr.SetLabels(map[string]string{clusterUIDLabel: string(clusterUID)}) + err = r.Update(ctx, &jtr) + if err != nil { + r.updateStatus(ctx, jtr, "Failed update JoinTokenRequest with cluster UID label") + return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err + } logger.Info("Reconciling") pod, err := util.FindStatefulSetPod(ctx, r.ClientSet, km.GetStatefulSetName(jtr.Spec.ClusterRef.Name), jtr.Spec.ClusterRef.Namespace) diff --git a/internal/controller/k0smotron.io/k0smotroncluster_controller.go b/internal/controller/k0smotron.io/k0smotroncluster_controller.go index 45fa7ebfc..842864e22 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_controller.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_controller.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" @@ -52,6 +53,11 @@ type ClusterReconciler struct { Recorder record.EventRecorder } +const ( + clusterUIDLabel = "k0smotron.io/cluster-uid" + clusterFinalizer = "k0smotron.io/finalizer" +) + //+kubebuilder:rbac:groups=k0smotron.io,resources=clusters,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=k0smotron.io,resources=clusters/status,verbs=get;update;patch //+kubebuilder:rbac:groups=k0smotron.io,resources=clusters/finalizers,verbs=update @@ -90,7 +96,39 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct logger.Info("Reconciling") if !kmc.ObjectMeta.DeletionTimestamp.IsZero() { - logger.Info("Cluster is being deleted, no action needed") + logger.Info("Cluster is being deleted") + if controllerutil.ContainsFinalizer(&kmc, clusterFinalizer) { + // Even if there is an error the finalizer must be removed for a complete removal of the + // cluster resource. In the worst case, the associated JointTokenRequest is not deleted. + defer func() { + controllerutil.RemoveFinalizer(&kmc, clusterFinalizer) + if err := r.Update(ctx, &kmc); err != nil { + logger.Error(err, "Error removing cluster finalizer") + } + }() + + // Once the cluster enters Terminating state, we ensure that the resources dependent on it + // are also removed. + // Note: owner references cannot be used in this case because JoinTokenRequest can be in a + // different namespace. + jtrl := &km.JoinTokenRequestList{} + err := r.List(ctx, jtrl, + client.MatchingLabels{ + clusterUIDLabel: string(kmc.GetUID()), + }) + if err != nil { + logger.Error(err, "Error retrieving JoinTokenRequests resources related to cluster") + return ctrl.Result{}, nil + } + for i := range jtrl.Items { + err := r.Delete(ctx, &jtrl.Items[i]) + if err != nil { + logger.Error(err, "Error removing JoinTokenRequests") + return ctrl.Result{}, nil + } + } + } + return ctrl.Result{}, nil } diff --git a/inttest/jointoken/jointoken_test.go b/inttest/jointoken/jointoken_test.go index bac9e4a56..4ecf828b0 100644 --- a/inttest/jointoken/jointoken_test.go +++ b/inttest/jointoken/jointoken_test.go @@ -19,12 +19,14 @@ package jointoken import ( "context" "testing" + "time" "github.com/k0sproject/k0s/inttest/common" "github.com/k0sproject/k0s/pkg/kubernetes/watch" "github.com/stretchr/testify/suite" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "github.com/k0sproject/k0smotron/inttest/util" @@ -61,6 +63,20 @@ func (s *JoinTokenSuite) TestK0sGetsUp() { s.createJoinTokenRequest(s.Context(), kc) s.Require().NoError(s.WaitForSecret(s.Context(), kc, "jtr-test", "jtr-test")) + + s.T().Log("removing cluster") + s.deleteK0smotronCluster(s.Context(), kc) + + s.T().Log("checking if JoinTokenRequest is deleted") + err = wait.PollUntilContextCancel(s.Context(), 100*time.Millisecond, true, func(ctx context.Context) (bool, error) { + res := kc.RESTClient().Get().AbsPath("/apis/k0smotron.io/v1beta1/namespaces/jtr-test/jointokenrequests/jtr-test").Do(s.Context()) + + var statusCode int + res.StatusCode(&statusCode) + + return statusCode == 404, nil + }) + s.Require().NoError(err) } func TestJoinTokenSuite(t *testing.T) { @@ -135,6 +151,11 @@ func (s *JoinTokenSuite) createJoinTokenRequest(ctx context.Context, kc *kuberne s.Require().NoError(res.Error()) } +func (s *JoinTokenSuite) deleteK0smotronCluster(ctx context.Context, kc *kubernetes.Clientset) { + res := kc.RESTClient().Delete().AbsPath("/apis/k0smotron.io/v1beta1/namespaces/kmc-test/clusters/kmc-test").Do(ctx) + s.Require().NoError(res.Error()) +} + func (s *JoinTokenSuite) WaitForSecret(ctx context.Context, clients kubernetes.Interface, name, namespace string) error { return watch.FromClient[*corev1.SecretList, corev1.Secret](clients.CoreV1().Secrets(namespace)). WithObjectName(name).