From 22ba7bf3d0cdf6f2a85d669471d436f34d830b11 Mon Sep 17 00:00:00 2001 From: Pankti Shah Date: Tue, 3 Oct 2023 14:52:14 -0700 Subject: [PATCH] Waiting for control plane to be fully upgraded before upgrading other components --- pkg/clusterapi/constants.go | 1 + pkg/controller/clusters/clusterapi.go | 31 +++++- pkg/controller/clusters/clusterapi_test.go | 114 ++++++++++++++++++++- 3 files changed, 142 insertions(+), 4 deletions(-) diff --git a/pkg/clusterapi/constants.go b/pkg/clusterapi/constants.go index 95929199c4f1..ab3d4257a6e3 100644 --- a/pkg/clusterapi/constants.go +++ b/pkg/clusterapi/constants.go @@ -4,4 +4,5 @@ import clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" const ( ControlPlaneReadyCondition clusterv1.ConditionType = "ControlPlaneReady" + ReadyCondition clusterv1.ConditionType = "Ready" ) diff --git a/pkg/controller/clusters/clusterapi.go b/pkg/controller/clusters/clusterapi.go index ceddd76aeb2c..4070349e5ee2 100644 --- a/pkg/controller/clusters/clusterapi.go +++ b/pkg/controller/clusters/clusterapi.go @@ -11,13 +11,40 @@ import ( anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/clusterapi" "github.com/aws/eks-anywhere/pkg/controller" + "github.com/aws/eks-anywhere/pkg/features" ) -// CheckControlPlaneReady is a controller helper to check whether a CAPI cluster CP for -// an eks-a cluster is ready or not. This is intended to be used from cluster reconcilers +// CheckControlPlaneReady is a controller helper to check whether KCP object for +// the cluster is ready or not. This is intended to be used from cluster reconcilers // due its signature and that it returns controller results with appropriate wait times whenever // the cluster is not ready. func CheckControlPlaneReady(ctx context.Context, client client.Client, log logr.Logger, cluster *anywherev1.Cluster) (controller.Result, error) { + if features.IsActive(features.ExperimentalSelfManagedClusterUpgrade()) { + kcp, err := controller.GetKubeadmControlPlane(ctx, client, cluster) + if err != nil { + return controller.Result{}, err + } + + if kcp == nil { + log.Info("KCP does not exist yet, requeuing") + return controller.ResultWithRequeue(5 * time.Second), nil + } + + // We make sure to check that the status is up to date before using it + if kcp.Status.ObservedGeneration != kcp.ObjectMeta.Generation { + log.Info("KCP information is outdated, requeing") + return controller.ResultWithRequeue(5 * time.Second), nil + } + + if !conditions.IsTrue(kcp, clusterapi.ReadyCondition) { + log.Info("KCP is not ready yet, requeing") + return controller.ResultWithRequeue(30 * time.Second), nil + } + + log.Info("KCP is ready") + return controller.Result{}, nil + } + capiCluster, err := controller.GetCAPICluster(ctx, client, cluster) if err != nil { return controller.Result{}, err diff --git a/pkg/controller/clusters/clusterapi_test.go b/pkg/controller/clusters/clusterapi_test.go index 6fc0ecb9ed51..f59d2c65bbd3 100644 --- a/pkg/controller/clusters/clusterapi_test.go +++ b/pkg/controller/clusters/clusterapi_test.go @@ -2,6 +2,7 @@ package clusters_test import ( "context" + "os" "testing" "time" @@ -10,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -17,8 +19,10 @@ import ( _ "github.com/aws/eks-anywhere/internal/test/envtest" anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/clusterapi" + "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/pkg/controller" "github.com/aws/eks-anywhere/pkg/controller/clusters" + "github.com/aws/eks-anywhere/pkg/features" ) func TestCheckControlPlaneReadyItIsReady(t *testing.T) { @@ -82,6 +86,92 @@ func TestCheckControlPlaneReadyErrorReading(t *testing.T) { g.Expect(err).To(MatchError(ContainSubstring("no kind is registered for the type"))) } +func TestCheckControlPlaneReadyItIsReadyWithKindlessUpgrade(t *testing.T) { + features.ClearCache() + os.Setenv(features.ExperimentalSelfManagedClusterUpgradeEnvVar, "true") + + g := NewWithT(t) + ctx := context.Background() + eksaCluster := eksaCluster() + kcp := kcpObject(func(k *v1beta1.KubeadmControlPlane) { + k.Status.Conditions = clusterv1.Conditions{ + { + Type: clusterapi.ReadyCondition, + Status: corev1.ConditionTrue, + }, + } + }) + + client := fake.NewClientBuilder().WithObjects(eksaCluster, kcp).Build() + + result, err := clusters.CheckControlPlaneReady(ctx, client, test.NewNullLogger(), eksaCluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result).To(Equal(controller.Result{})) +} + +func TestCheckControlPlaneReadyNoKcpWithKindlessUpgrade(t *testing.T) { + features.ClearCache() + os.Setenv(features.ExperimentalSelfManagedClusterUpgradeEnvVar, "true") + + g := NewWithT(t) + ctx := context.Background() + eksaCluster := eksaCluster() + client := fake.NewClientBuilder().WithObjects(eksaCluster).Build() + + result, err := clusters.CheckControlPlaneReady(ctx, client, test.NewNullLogger(), eksaCluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result).To(Equal( + controller.Result{Result: &controllerruntime.Result{RequeueAfter: 5 * time.Second}}), + ) +} + +func TestCheckControlPlaneNotReadyWithKindlessUpgrade(t *testing.T) { + features.ClearCache() + os.Setenv(features.ExperimentalSelfManagedClusterUpgradeEnvVar, "true") + + g := NewWithT(t) + ctx := context.Background() + eksaCluster := eksaCluster() + kcp := kcpObject(func(k *v1beta1.KubeadmControlPlane) { + k.Status = v1beta1.KubeadmControlPlaneStatus{ + ObservedGeneration: 2, + } + }) + + client := fake.NewClientBuilder().WithObjects(eksaCluster, kcp).Build() + + result, err := clusters.CheckControlPlaneReady(ctx, client, test.NewNullLogger(), eksaCluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result).To(Equal( + controller.Result{Result: &controllerruntime.Result{RequeueAfter: 5 * time.Second}}), + ) +} + +func TestCheckControlPlaneStatusNotReadyWithKindlessUpgrade(t *testing.T) { + features.ClearCache() + os.Setenv(features.ExperimentalSelfManagedClusterUpgradeEnvVar, "true") + + g := NewWithT(t) + ctx := context.Background() + eksaCluster := eksaCluster() + kcp := kcpObject(func(k *v1beta1.KubeadmControlPlane) { + k.Status.Conditions = clusterv1.Conditions{ + { + Type: clusterapi.ReadyCondition, + Status: corev1.ConditionFalse, + }, + } + }) + + client := fake.NewClientBuilder().WithObjects(eksaCluster, kcp).Build() + + result, err := clusters.CheckControlPlaneReady(ctx, client, test.NewNullLogger(), eksaCluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result).To(Equal( + controller.Result{Result: &controllerruntime.Result{RequeueAfter: 30 * time.Second}}), + ) +} + func eksaCluster() *anywherev1.Cluster { return &anywherev1.Cluster{ TypeMeta: metav1.TypeMeta{ @@ -104,13 +194,33 @@ func capiCluster(opts ...capiClusterOpt) *clusterv1.Cluster { }, ObjectMeta: metav1.ObjectMeta{ Name: "my-cluster", - Namespace: "eksa-system", + Namespace: constants.EksaSystemNamespace, }, } - for _, opt := range opts { opt(c) } return c } + +type kcpObjectOpt func(*v1beta1.KubeadmControlPlane) + +func kcpObject(opts ...kcpObjectOpt) *v1beta1.KubeadmControlPlane { + k := &v1beta1.KubeadmControlPlane{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeadmControlPlane", + APIVersion: v1beta1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: constants.EksaSystemNamespace, + }, + } + + for _, opt := range opts { + opt(k) + } + + return k +}