diff --git a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go index 198545f55..82191c94f 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go @@ -207,14 +207,14 @@ func (r *ClusterReconciler) reconcileEtcdStatefulSet(ctx context.Context, kmc *k } } - statefulSet := r.generateEtcdStatefulSet(kmc, desiredReplicas) + statefulSet := r.generateEtcdStatefulSet(kmc, foundStatefulSet, desiredReplicas) _ = ctrl.SetControllerReference(kmc, &statefulSet, r.Scheme) return r.Client.Patch(ctx, &statefulSet, client.Apply, patchOpts...) } -func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas int32) apps.StatefulSet { +func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, existingSts *apps.StatefulSet, replicas int32) apps.StatefulSet { labels := labelsForEtcdCluster(kmc) size := kmc.Spec.Etcd.Persistence.Size @@ -329,7 +329,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in SecurityContext: &v1.PodSecurityContext{ FSGroup: ptr.To(int64(1001)), }, - InitContainers: r.generateEtcdInitContainers(kmc), + InitContainers: r.generateEtcdInitContainers(kmc, existingSts), Containers: []v1.Container{{ Name: "etcd", Image: kmc.Spec.Etcd.Image, @@ -393,13 +393,22 @@ func (r *ClusterReconciler) initialCluster(kmc *km.Cluster, replicas int32) stri return strings.Join(members, ",") } -func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster) []v1.Container { +func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster, existingSts *apps.StatefulSet) []v1.Container { + checkImage := kmc.Spec.GetImage() + if existingSts != nil { + for _, c := range existingSts.Spec.Template.Spec.InitContainers { + if c.Name == "dns-check" { + checkImage = c.Image + break + } + } + } return []v1.Container{ { // Wait for the pods dns name is resolvable, since it takes some time after the pod is created // and etcd tries to connect to the other members using the dns names Name: "dns-check", - Image: kmc.Spec.GetImage(), + Image: checkImage, ImagePullPolicy: v1.PullIfNotPresent, Command: []string{"/bin/sh", "-c"}, Args: []string{"getent ahostsv4 ${HOSTNAME}.${SVC_NAME}." + kmc.Namespace + ".svc"}, @@ -480,7 +489,7 @@ set -eu export ETCDCTL_ENDPOINTS=https://${SVC_NAME}:2379 -if [[ ! -f /var/lib/k0s/etcd/snap/db ]]; then +if [[ ! -f /var/lib/k0s/etcd/member/snap/db ]]; then echo "Checking if cluster is functional" if etcdctl member list; then echo "Cluster is functional" diff --git a/internal/controller/k0smotron.io/k0smotroncluster_etcd_test.go b/internal/controller/k0smotron.io/k0smotroncluster_etcd_test.go index 198886dbb..d5ee1deb8 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_etcd_test.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_etcd_test.go @@ -82,7 +82,7 @@ func TestEtcd_generateEtcdStatefulSet(t *testing.T) { for _, tc := range tests { t.Run("", func(t *testing.T) { r := new(ClusterReconciler) - sts := r.generateEtcdStatefulSet(tc.cluster, 1) + sts := r.generateEtcdStatefulSet(tc.cluster, nil, 1) for _, w := range tc.want { assert.True(t, strings.Contains(sts.Spec.Template.Spec.Containers[0].Args[1], w)) } diff --git a/inttest/ha-controller-etcd/ha_controller_etcd_test.go b/inttest/ha-controller-etcd/ha_controller_etcd_test.go index 998600683..982d605b0 100644 --- a/inttest/ha-controller-etcd/ha_controller_etcd_test.go +++ b/inttest/ha-controller-etcd/ha_controller_etcd_test.go @@ -18,14 +18,22 @@ package hacontrolleretcd import ( "context" + "k8s.io/apimachinery/pkg/util/wait" "testing" + "time" "github.com/k0sproject/k0s/inttest/common" - "github.com/k0sproject/k0smotron/inttest/util" + km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" "github.com/stretchr/testify/suite" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + + "github.com/k0sproject/k0smotron/inttest/util" ) type HAControllerEtcdSuite struct { @@ -77,6 +85,20 @@ func (s *HAControllerEtcdSuite) TestK0sGetsUp() { s.Require().NoError(err) s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC)) + s.T().Log("update cluster") + s.updateK0smotronCluster(s.Context(), rc) + + err = wait.PollUntilContextCancel(s.Context(), 5*time.Second, true, func(ctx context.Context) (bool, error) { + sts, err := kc.AppsV1().StatefulSets("kmc-test").Get(s.Context(), "kmc-kmc-test", metav1.GetOptions{}) + if err != nil { + return false, nil + } + + return sts.Spec.Template.Spec.Containers[0].Image == "k0sproject/k0s:v1.28.3-k0s.0", nil + }) + s.Require().NoError(err) + + s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test")) } func TestHAControllerEtcdSuite(t *testing.T) { @@ -110,6 +132,7 @@ func (s *HAControllerEtcdSuite) createK0smotronCluster(ctx context.Context, kc * }, "spec": { "replicas": 3, + "version": "v1.27.2-k0s.0", "service":{ "type": "NodePort" }, @@ -128,6 +151,26 @@ func (s *HAControllerEtcdSuite) createK0smotronCluster(ctx context.Context, kc * s.Require().NoError(res.Error()) } +func (s *HAControllerEtcdSuite) updateK0smotronCluster(ctx context.Context, rc *rest.Config) { + crdConfig := *rc + crdConfig.ContentConfig.GroupVersion = &km.GroupVersion + crdConfig.APIPath = "/apis" + crdConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme) + crdConfig.UserAgent = rest.DefaultKubernetesUserAgent() + crdRestClient, err := rest.UnversionedRESTClientFor(&crdConfig) + s.Require().NoError(err) + + patch := `[{"op": "replace", "path": "/spec/version", "value": "v1.28.3-k0s.0"}]` + res := crdRestClient. + Patch(types.JSONPatchType). + Resource("clusters"). + Name("kmc-test"). + Namespace("kmc-test"). + Body([]byte(patch)). + Do(ctx) + s.Require().NoError(res.Error()) +} + func (s *HAControllerEtcdSuite) getPod(ctx context.Context, kc *kubernetes.Clientset) corev1.Pod { pods, err := kc.CoreV1().Pods("kmc-test").List( ctx,