Update HA k0smotron cluster fix & test

internal/controller/k0smotron.io/k0smotroncluster_etcd.go

@@ -207,14 +207,14 @@ func (r *ClusterReconciler) reconcileEtcdStatefulSet(ctx context.Context, kmc *k
 		}
 	}
 
-	statefulSet := r.generateEtcdStatefulSet(kmc, desiredReplicas)
+	statefulSet := r.generateEtcdStatefulSet(kmc, foundStatefulSet, desiredReplicas)
 
 	_ = ctrl.SetControllerReference(kmc, &statefulSet, r.Scheme)
 
 	return r.Client.Patch(ctx, &statefulSet, client.Apply, patchOpts...)
 }
 
-func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas int32) apps.StatefulSet {
+func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, existingSts *apps.StatefulSet, replicas int32) apps.StatefulSet {
 	labels := labelsForEtcdCluster(kmc)
 
 	size := kmc.Spec.Etcd.Persistence.Size
@@ -329,7 +329,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in
 					SecurityContext: &v1.PodSecurityContext{
 						FSGroup: ptr.To(int64(1001)),
 					},
-					InitContainers: r.generateEtcdInitContainers(kmc),
+					InitContainers: r.generateEtcdInitContainers(kmc, existingSts),
 					Containers: []v1.Container{{
 						Name:            "etcd",
 						Image:           kmc.Spec.Etcd.Image,
@@ -393,13 +393,22 @@ func (r *ClusterReconciler) initialCluster(kmc *km.Cluster, replicas int32) stri
 	return strings.Join(members, ",")
 }
 
-func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster) []v1.Container {
+func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster, existingSts *apps.StatefulSet) []v1.Container {
+	checkImage := kmc.Spec.GetImage()
+	if existingSts != nil {
+		for _, c := range existingSts.Spec.Template.Spec.InitContainers {
+			if c.Name == "dns-check" {
+				checkImage = c.Image
+				break
+			}
+		}
+	}
 	return []v1.Container{
 		{
 			// Wait for the pods dns name is resolvable, since it takes some time after the pod is created
 			// and etcd tries to connect to the other members using the dns names
 			Name:            "dns-check",
-			Image:           kmc.Spec.GetImage(),
+			Image:           checkImage,
 			ImagePullPolicy: v1.PullIfNotPresent,
 			Command:         []string{"/bin/sh", "-c"},
 			Args:            []string{"getent ahostsv4 ${HOSTNAME}.${SVC_NAME}." + kmc.Namespace + ".svc"},
@@ -480,7 +489,7 @@ set -eu
 
 export ETCDCTL_ENDPOINTS=https://${SVC_NAME}:2379
 
-if [[ ! -f /var/lib/k0s/etcd/snap/db ]]; then
+if [[ ! -f /var/lib/k0s/etcd/member/snap/db ]]; then
   echo "Checking if cluster is functional"
   if etcdctl member list; then
     echo "Cluster is functional"

internal/controller/k0smotron.io/k0smotroncluster_etcd_test.go

@@ -82,7 +82,7 @@ func TestEtcd_generateEtcdStatefulSet(t *testing.T) {
 	for _, tc := range tests {
 		t.Run("", func(t *testing.T) {
 			r := new(ClusterReconciler)
-			sts := r.generateEtcdStatefulSet(tc.cluster, 1)
+			sts := r.generateEtcdStatefulSet(tc.cluster, nil, 1)
 			for _, w := range tc.want {
 				assert.True(t, strings.Contains(sts.Spec.Template.Spec.Containers[0].Args[1], w))
 			}

inttest/ha-controller-etcd/ha_controller_etcd_test.go

@@ -18,14 +18,22 @@ package hacontrolleretcd
 
 import (
 	"context"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"testing"
+	"time"
 
 	"github.com/k0sproject/k0s/inttest/common"
-	"github.com/k0sproject/k0smotron/inttest/util"
+	km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1"
 	"github.com/stretchr/testify/suite"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	"github.com/k0sproject/k0smotron/inttest/util"
 )
 
 type HAControllerEtcdSuite struct {
@@ -77,6 +85,20 @@ func (s *HAControllerEtcdSuite) TestK0sGetsUp() {
 	s.Require().NoError(err)
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))
 
+	s.T().Log("update cluster")
+	s.updateK0smotronCluster(s.Context(), rc)
+
+	err = wait.PollUntilContextCancel(s.Context(), 5*time.Second, true, func(ctx context.Context) (bool, error) {
+		sts, err := kc.AppsV1().StatefulSets("kmc-test").Get(s.Context(), "kmc-kmc-test", metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+
+		return sts.Spec.Template.Spec.Containers[0].Image == "k0sproject/k0s:v1.28.3-k0s.0", nil
+	})
+	s.Require().NoError(err)
+
+	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 }
 
 func TestHAControllerEtcdSuite(t *testing.T) {
@@ -110,6 +132,7 @@ func (s *HAControllerEtcdSuite) createK0smotronCluster(ctx context.Context, kc *
 		},
 		"spec": {
 		    "replicas": 3,
+			"version": "v1.27.2-k0s.0",
 			"service":{
 				"type": "NodePort"
 			},
@@ -128,6 +151,26 @@ func (s *HAControllerEtcdSuite) createK0smotronCluster(ctx context.Context, kc *
 	s.Require().NoError(res.Error())
 }
 
+func (s *HAControllerEtcdSuite) updateK0smotronCluster(ctx context.Context, rc *rest.Config) {
+	crdConfig := *rc
+	crdConfig.ContentConfig.GroupVersion = &km.GroupVersion
+	crdConfig.APIPath = "/apis"
+	crdConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme)
+	crdConfig.UserAgent = rest.DefaultKubernetesUserAgent()
+	crdRestClient, err := rest.UnversionedRESTClientFor(&crdConfig)
+	s.Require().NoError(err)
+
+	patch := `[{"op": "replace", "path": "/spec/version", "value": "v1.28.3-k0s.0"}]`
+	res := crdRestClient.
+		Patch(types.JSONPatchType).
+		Resource("clusters").
+		Name("kmc-test").
+		Namespace("kmc-test").
+		Body([]byte(patch)).
+		Do(ctx)
+	s.Require().NoError(res.Error())
+}
+
 func (s *HAControllerEtcdSuite) getPod(ctx context.Context, kc *kubernetes.Clientset) corev1.Pod {
 	pods, err := kc.CoreV1().Pods("kmc-test").List(
 		ctx,