diff --git a/api/controlplane/v1beta1/groupversion_info.go b/api/controlplane/v1beta1/groupversion_info.go index 416aba9cc..0a66649d4 100644 --- a/api/controlplane/v1beta1/groupversion_info.go +++ b/api/controlplane/v1beta1/groupversion_info.go @@ -34,3 +34,5 @@ var ( // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) + +const K0sClusterIDAnnotation = "k0sproject.io/cluster-id" diff --git a/internal/controller/controlplane/k0s_controlplane_controller.go b/internal/controller/controlplane/k0s_controlplane_controller.go index a0fb8af80..110b87235 100644 --- a/internal/controller/controlplane/k0s_controlplane_controller.go +++ b/internal/controller/controlplane/k0s_controlplane_controller.go @@ -151,11 +151,18 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct } log.Info("Status updated successfully") + if kcp.Status.ControlPlaneReady { + if perr := c.Client.Patch(ctx, cluster, client.Merge); perr != nil { + err = fmt.Errorf("failed to patch cluster: %w", perr) + } + } + // Requeue the reconciliation if the status is not ready if !kcp.Status.Ready { log.Info("Requeuing reconciliation in 20sec since the control plane is not ready") res = ctrl.Result{RequeueAfter: 20 * time.Second, Requeue: true} } + }() log = log.WithValues("cluster", cluster.Name) diff --git a/internal/controller/controlplane/k0smotron_controlplane_controller.go b/internal/controller/controlplane/k0smotron_controlplane_controller.go index 9678ab762..5687b67bf 100644 --- a/internal/controller/controlplane/k0smotron_controlplane_controller.go +++ b/internal/controller/controlplane/k0smotron_controlplane_controller.go @@ -22,11 +22,13 @@ import ( "reflect" "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "sigs.k8s.io/cluster-api/controllers/remote" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -118,6 +120,29 @@ func (c *K0smotronController) Reconcile(ctx context.Context, req ctrl.Request) ( } } + if ready { + remoteClient, err := remote.NewClusterClient(ctx, "k0smotron", c.Client, util.ObjectKey(cluster)) + if err != nil { + return res, fmt.Errorf("failed to create remote client: %w", err) + } + + ns := &corev1.Namespace{} + err = remoteClient.Get(ctx, types.NamespacedName{Name: "kube-system", Namespace: ""}, ns) + if err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 30}, nil + } + return res, fmt.Errorf("failed to get namespace: %w", err) + } + + annotations.AddAnnotations(cluster, map[string]string{ + cpv1beta1.K0sClusterIDAnnotation: fmt.Sprintf("kube-system:%s", ns.GetUID()), + }) + if err := c.Client.Patch(ctx, cluster, client.Merge); err != nil { + return res, fmt.Errorf("failed to patch cluster: %w", err) + } + } + // TODO: We need to have bit more detailed status and conditions handling kcp.Status.Ready = ready kcp.Status.ExternalManagedControlPlane = true diff --git a/internal/controller/controlplane/status.go b/internal/controller/controlplane/status.go index 3e4589afb..b984eb951 100644 --- a/internal/controller/controlplane/status.go +++ b/internal/controller/controlplane/status.go @@ -27,6 +27,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/remote" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/collections" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/controller-runtime/pkg/log" @@ -185,6 +186,11 @@ func (c *K0sController) updateStatus(ctx context.Context, kcp *cpv1beta1.K0sCont kcp.Status.ControlPlaneReady = true kcp.Status.Inititalized = true + // Set the k0s cluster ID annotation + annotations.AddAnnotations(cluster, map[string]string{ + cpv1beta1.K0sClusterIDAnnotation: fmt.Sprintf("kube-system:%s", ns.GetUID()), + }) + return nil } diff --git a/inttest/capi-controlplane-docker/capi_controlplane_docker_test.go b/inttest/capi-controlplane-docker/capi_controlplane_docker_test.go index 603be7cd8..705bf00c7 100644 --- a/inttest/capi-controlplane-docker/capi_controlplane_docker_test.go +++ b/inttest/capi-controlplane-docker/capi_controlplane_docker_test.go @@ -19,8 +19,10 @@ package capicontolplanedocker import ( "context" "fmt" + controlplanev1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1" "os" "os/exec" + "sigs.k8s.io/cluster-api/api/v1beta1" "strconv" "strings" "testing" @@ -102,8 +104,7 @@ func (s *CAPIControlPlaneDockerSuite) TestCAPIControlPlaneDocker() { kmcKC, err := util.GetKMCClientSet(s.ctx, s.client, "docker-test-cluster", "default", localPort) s.Require().NoError(err) - // nolint:staticcheck - err = wait.PollImmediateUntilWithContext(s.ctx, 1*time.Second, func(ctx context.Context) (bool, error) { + err = wait.PollUntilContextCancel(s.ctx, 1*time.Second, true, func(ctx context.Context) (bool, error) { b, _ := s.client.RESTClient(). Get(). AbsPath("/healthz"). @@ -113,6 +114,19 @@ func (s *CAPIControlPlaneDockerSuite) TestCAPIControlPlaneDocker() { }) s.Require().NoError(err) + err = wait.PollUntilContextCancel(s.ctx, 1*time.Second, true, func(ctx context.Context) (bool, error) { + var cluster v1beta1.Cluster + err = s.client.RESTClient(). + Get(). + AbsPath("/apis/cluster.x-k8s.io/v1beta1/namespaces/default/clusters/docker-test-cluster"). + Do(ctx). + Into(&cluster) + + clusterIDAnnotation, found := cluster.GetAnnotations()[controlplanev1beta1.K0sClusterIDAnnotation] + return found && strings.Contains(clusterIDAnnotation, "kube-system"), nil + }) + s.Require().NoError(err) + for i := 0; i < 3; i++ { // nolint:staticcheck err = wait.PollImmediateUntilWithContext(s.ctx, 1*time.Second, func(ctx context.Context) (bool, error) { diff --git a/inttest/capi-docker/capi_docker_test.go b/inttest/capi-docker/capi_docker_test.go index 2814eb478..3de81b1f4 100644 --- a/inttest/capi-docker/capi_docker_test.go +++ b/inttest/capi-docker/capi_docker_test.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "os/exec" + "sigs.k8s.io/cluster-api/api/v1beta1" "strings" "testing" "time" @@ -100,6 +101,7 @@ func (s *CAPIDockerSuite) TestCAPIDocker() { s.Require().NoError(util.WaitForStatefulSet(s.ctx, s.client, "kmc-docker-test", "default")) s.checkControlPlaneStatus(s.ctx, s.restConfig) + s.checkClusterIDAnnotation(s.ctx) s.T().Log("Starting portforward") fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-test-0", "default", 30443) @@ -122,7 +124,7 @@ func (s *CAPIDockerSuite) TestCAPIDocker() { s.Require().NoError(util.WaitForNodeReadyStatus(s.ctx, kmcKC, "docker-test-0", corev1.ConditionTrue)) node, err := kmcKC.CoreV1().Nodes().Get(s.ctx, "docker-test-0", metav1.GetOptions{}) s.Require().NoError(err) - s.Require().Equal("v1.27.1+k0s", node.Status.NodeInfo.KubeletVersion) + s.Require().Equal("v1.31.1+k0s", node.Status.NodeInfo.KubeletVersion) fooLabel, ok := node.Labels["k0sproject.io/foo"] s.Require().True(ok) s.Require().Equal("bar", fooLabel) @@ -141,6 +143,7 @@ func (s *CAPIDockerSuite) TestCAPIDocker() { extraFile, err := getDockerNodeFile("docker-test-0", "/tmp/test-file") s.Require().NoError(err) s.Require().Equal("test-file", extraFile) + } func (s *CAPIDockerSuite) prepareCerts() { @@ -183,8 +186,7 @@ func (s *CAPIDockerSuite) checkControlPlaneStatus(ctx context.Context, rc *rest. crdRestClient, err := rest.UnversionedRESTClientFor(&crdConfig) s.Require().NoError(err) - // nolint:staticcheck - err = wait.PollImmediateUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (bool, error) { + err = wait.PollUntilContextCancel(ctx, 1*time.Second, true, func(ctx context.Context) (bool, error) { var kcp controlplanev1beta1.K0smotronControlPlane err = crdRestClient. Get(). @@ -200,6 +202,23 @@ func (s *CAPIDockerSuite) checkControlPlaneStatus(ctx context.Context, rc *rest. s.Require().NoError(err) } +func (s *CAPIDockerSuite) checkClusterIDAnnotation(ctx context.Context) { + err := wait.PollUntilContextCancel(ctx, 1*time.Second, true, func(ctx context.Context) (bool, error) { + var cluster v1beta1.Cluster + _ = s.client.RESTClient(). + Get(). + AbsPath("/apis/cluster.x-k8s.io/v1beta1/namespaces/default/clusters/docker-test"). + Do(ctx). + Into(&cluster) + + s.T().Log(cluster) + clusterIDAnnotation, found := cluster.GetAnnotations()[controlplanev1beta1.K0sClusterIDAnnotation] + return found && strings.Contains(clusterIDAnnotation, "kube-system"), nil + }) + + s.Require().NoError(err) +} + func getDockerNodeFile(nodeName string, path string) (string, error) { output, err := exec.Command("docker", "exec", nodeName, "cat", path).Output() if err != nil { @@ -238,7 +257,7 @@ kind: K0smotronControlPlane metadata: name: docker-test-cp spec: - version: v1.27.2-k0s.0 + version: v1.31.2-k0s.0 certificateRefs: - name: docker-test-ca type: ca @@ -279,7 +298,7 @@ metadata: name: docker-test-0 namespace: default spec: - version: v1.27.1 + version: v1.31.1 clusterName: docker-test bootstrap: configRef: @@ -298,7 +317,7 @@ metadata: namespace: default spec: # version is deliberately different to be able to verify we actually pick it up :) - version: v1.27.1+k0s.0 + version: v1.31.1+k0s.0 args: - --labels=k0sproject.io/foo=bar preStartCommands: