Prevent distro and backingstore switch (loft-sh#1668)

* feat: Annotate initial distro and backing store type to config secret

Change-Id: I4f1068842002b2ccd0485741115a9cef5f9b933e
Signed-off-by: Thomas Kosiewski <thomas.kosiewski@loft.sh>

* refactor: Use helm history and heuristics to determine distro and backing storage type

Change-Id: I7310910b61617d382f220834bf0366815c0408cd
Signed-off-by: Thomas Kosiewski <thomas.kosiewski@loft.sh>

* refactor: helm & logic improvements

* refactor: helm & logic improvements

* fix: Added missing error return

Change-Id: I504a2dd42484e5a9c9470854ecebcb15d7ce9a6e
Signed-off-by: Thomas Kosiewski <thomas.kosiewski@loft.sh>

---------

Signed-off-by: Thomas Kosiewski <thomas.kosiewski@loft.sh>
Co-authored-by: Thomas Kosiewski <thomas.kosiewski@loft.sh>
Co-authored-by: Fabian Kramm <fab.kramm@googlemail.com>

.github/workflows/e2e.yaml

@@ -13,7 +13,7 @@ on:
       - "!**.md"
       - "Dockerfile.release"
       - ".github/workflows/e2e.yaml"
-      - "charts/**"
+      - "chart/**"
       - "manifests/**"
 
 concurrency:

.github/workflows/unit-tests.yaml

@@ -11,7 +11,7 @@ on:
       - "!test/**" # exclude changes in e2e tests
       - ".github/workflows/unit-tests.yaml"
       - "hack/test.sh"
-      - "charts/**"
+      - "chart/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.gitignore

@@ -11,6 +11,7 @@ profile.out
 /release
 /cmd/vclusterctl/cmd/charts/
 /pkg/embed/charts/vcluster-*
+/pkg/embed/chart/vcluster-*
 /dist
 *.test
 tests/__snapshot__

.goreleaser.yaml

@@ -3,7 +3,7 @@ project_name: vcluster
 before:
   hooks:
     - go mod tidy
-    - just embed-charts {{ .Version }}
+    - just embed-chart {{ .Version }}
     - just clean-release
     - just copy-assets
     - just generate-vcluster-images {{ .Version }}
@@ -29,7 +29,7 @@ builds:
       - -mod
       - vendor
     tags:
-      - embed_charts
+      - embed_chart
     ldflags:
       - -s -w
       - -X github.com/loft-sh/vcluster/pkg/telemetry.SyncerVersion={{.Version}}
@@ -64,7 +64,7 @@ builds:
       - -mod
       - vendor
     tags:
-      - embed_charts
+      - embed_chart
     ldflags:
       - -s -w
       - -X main.version={{.Version}}

Dockerfile.cli

@@ -18,9 +18,9 @@ COPY pkg/ pkg/
 COPY cmd/ cmd/
 
 # Copy and embed the helm charts
-COPY charts/ charts/
+COPY chart/ chart/
 COPY hack/ hack/
-RUN go generate -tags embed_charts ./...
+RUN go generate -tags embed_chart ./...
 
 ENV GO111MODULE on
 ENV DEBUG true
@@ -30,7 +30,7 @@ RUN mkdir -p /.cache
 ENV GOCACHE=/.cache
 
 # Build cmd
-RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -tags embed_charts -o /vcluster cmd/vclusterctl/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -tags embed_chart -o /vcluster cmd/vclusterctl/main.go
 
 # we use alpine for easier debugging
 FROM alpine:3.19

Justfile

@@ -75,10 +75,10 @@ generate-cli-docs:
 generate-config-schema:
   go run -mod vendor ./hack/schema/main.go
 
-# Embed the charts into the vcluster binary
+# Embed the chart into the vcluster binary
 [private]
-embed-charts version="0.0.0":
-  RELEASE_VERSION={{ version }} go generate -tags embed_charts ./...
+embed-chart version="0.0.0":
+  RELEASE_VERSION={{ version }} go generate -tags embed_chart ./...
 
 # Run e2e tests
 e2e distribution="k3s" path="./test/e2e" multinamespace="false": create-kind && delete-kind
@@ -107,7 +107,7 @@ e2e distribution="k3s" path="./test/e2e" multinamespace="false": create-kind &&
     --debug \
     --connect=false \
     --distro={{ distribution }} \
-    --local-chart-dir ./charts/{{ distribution }} \
+    --local-chart-dir ./chart/ \
     -f ./dist/commonValues.yaml \
     -f {{ path }}/values.yaml \
     $([[ "{{ multinamespace }}" = "true" ]] && echo "-f ./test/multins_values.yaml" || echo "")
@@ -122,8 +122,8 @@ e2e distribution="k3s" path="./test/e2e" multinamespace="false": create-kind &&
     go test -v -ginkgo.v -ginkgo.skip='.*NetworkPolicy.*' -ginkgo.fail-fast
 
 cli version="0.0.0" *ARGS="":
-  RELEASE_VERSION={{ version }} go generate -tags embed_charts ./...
-  go run -tags embed_charts -mod vendor -ldflags "-X main.version={{ version }}" ./cmd/vclusterctl/main.go {{ ARGS }}
+  RELEASE_VERSION={{ version }} go generate -tags embed_chart ./...
+  go run -tags embed_chart -mod vendor -ldflags "-X main.version={{ version }}" ./cmd/vclusterctl/main.go {{ ARGS }}
 
 # --- Docs ---
 

cmd/vcluster/cmd/start.go

@@ -70,7 +70,7 @@ func ExecuteStart(ctx context.Context, options *StartOptions) error {
 	}
 
 	// init config
-	err = setup.InitConfig(vConfig)
+	err = setup.InitAndValidateConfig(ctx, vConfig)
 	if err != nil {
 		return err
 	}

cmd/vclusterctl/cmd/create.go

@@ -300,6 +300,34 @@ func (cmd *CreateCmd) Run(ctx context.Context, args []string) error {
 		return err
 	}
 
+	// check if vcluster already exists
+	if !cmd.Upgrade {
+		release, err := helm.NewSecrets(cmd.kubeClient).Get(ctx, args[0], cmd.Namespace)
+		if err != nil && !kerrors.IsNotFound(err) {
+			return errors.Wrap(err, "get helm releases")
+		} else if release != nil &&
+			release.Chart != nil &&
+			release.Chart.Metadata != nil &&
+			(release.Chart.Metadata.Name == "vcluster" || release.Chart.Metadata.Name == "vcluster-k0s" || release.Chart.Metadata.Name == "vcluster-k8s" || release.Chart.Metadata.Name == "vcluster-eks") &&
+			release.Secret != nil &&
+			release.Secret.Labels != nil &&
+			release.Secret.Labels["status"] == "deployed" {
+			if cmd.Connect {
+				connectCmd := &ConnectCmd{
+					GlobalFlags:           cmd.GlobalFlags,
+					UpdateCurrent:         cmd.UpdateCurrent,
+					KubeConfigContextName: cmd.KubeConfigContextName,
+					KubeConfig:            "./kubeconfig.yaml",
+					Log:                   cmd.log,
+				}
+
+				return connectCmd.Connect(ctx, args[0], nil)
+			}
+
+			return fmt.Errorf("vcluster %s already exists in namespace %s\n- Use `vcluster create %s -n %s --upgrade` to upgrade the vcluster\n- Use `vcluster connect %s -n %s` to access the vcluster", args[0], cmd.Namespace, args[0], cmd.Namespace, args[0], cmd.Namespace)
+		}
+	}
+
 	// we have to upgrade / install the chart
 	err = cmd.deployChart(ctx, args[0], chartValues, helmBinaryPath)
 	if err != nil {
@@ -393,7 +421,7 @@ func (cmd *CreateCmd) deployChart(ctx context.Context, vClusterName, chartValues
 		if cmd.ChartVersion == upgrade.GetVersion() { // use embedded chart if default version
 			embeddedChartName := fmt.Sprintf("%s-%s.tgz", cmd.ChartName, upgrade.GetVersion())
 			// not using filepath.Join because the embed.FS separator is not OS specific
-			embeddedChartPath := fmt.Sprintf("charts/%s", embeddedChartName)
+			embeddedChartPath := fmt.Sprintf("chart/%s", embeddedChartName)
 			embeddedChartFile, err := embed.Charts.ReadFile(embeddedChartPath)
 			if err != nil && errors.Is(err, fs.ErrNotExist) {
 				cmd.log.Infof("Chart not embedded: %q, pulling from helm repository.", err)

config/config.go

@@ -102,6 +102,23 @@ func (c *Config) DecodeYAML(r io.Reader) error {
 	return nil
 }
 
+// BackingStoreType returns the backing store type of the vCluster.
+// If no backing store is enabled, it returns StoreTypeUnknown.
+func (c *Config) BackingStoreType() StoreType {
+	switch {
+	case c.ControlPlane.BackingStore.Etcd.Embedded.Enabled:
+		return StoreTypeEmbeddedEtcd
+	case c.ControlPlane.BackingStore.Etcd.Deploy.Enabled:
+		return StoreTypeExternalEtcd
+	case c.ControlPlane.BackingStore.Database.Embedded.Enabled:
+		return StoreTypeEmbeddedDatabase
+	case c.ControlPlane.BackingStore.Database.External.Enabled:
+		return StoreTypeExternalDatabase
+	default:
+		return StoreTypeEmbeddedDatabase
+	}
+}
+
 func (c *Config) Distro() string {
 	if c.ControlPlane.Distro.K3S.Enabled {
 		return K3SDistro

config/default_extra_values.go

@@ -15,6 +15,15 @@ const (
 	Unknown   = "unknown"
 )
 
+type StoreType string
+
+const (
+	StoreTypeEmbeddedEtcd     StoreType = "embedded-etcd"
+	StoreTypeExternalEtcd     StoreType = "external-etcd"
+	StoreTypeEmbeddedDatabase StoreType = "embedded-database"
+	StoreTypeExternalDatabase StoreType = "external-database"
+)
+
 // K3SVersionMap holds the supported k3s versions
 var K3SVersionMap = map[string]string{
 	"1.29": "rancher/k3s:v1.29.0-k3s1",

config/diff.go

@@ -9,10 +9,8 @@ import (
 	"github.com/ghodss/yaml"
 )
 
-var (
-	// ErrUnsupportedType is returned if the type is not implemented
-	ErrUnsupportedType = errors.New("unsupported type")
-)
+// ErrUnsupportedType is returned if the type is not implemented
+var ErrUnsupportedType = errors.New("unsupported type")
 
 func Diff(fromConfig *Config, toConfig *Config) (string, error) {
 	// convert to map[string]interface{}
@@ -151,11 +149,12 @@ func (f *StrBool) UnmarshalJSON(data []byte) error {
 }
 
 func (f *StrBool) MarshalJSON() ([]byte, error) {
-	if *f == "true" {
+	switch *f {
+	case "true":
 		return []byte("true"), nil
-	} else if *f == "false" {
+	case "false":
 		return []byte("false"), nil
+	default:
+		return []byte("\"" + *f + "\""), nil
 	}
-
-	return []byte("\"" + *f + "\""), nil
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/loft-sh/vcluster
 
-go 1.22.0
+go 1.22.1
 
 require (
 	github.com/blang/semver v3.5.1+incompatible

hack/embed-charts.sh → hack/embed-chart.sh

@@ -7,10 +7,10 @@ set -eu
 VCLUSTER_ROOT="$(dirname ${0})/.."
 RELEASE_VERSION="${RELEASE_VERSION:-0.0.1}"
 RELEASE_VERSION="${RELEASE_VERSION#"v"}" # remove "v" prefix
-EMBED_DIR="${VCLUSTER_ROOT}/pkg/embed/charts"
+EMBED_DIR="${VCLUSTER_ROOT}/pkg/embed/chart"
 
 rm -rfv "${EMBED_DIR}"
 mkdir "${EMBED_DIR}"
 
 touch "${EMBED_DIR}/gitkeep.tgz"
-helm package --version "${RELEASE_VERSION}" "${VCLUSTER_ROOT}/chart" -d "${EMBED_DIR}";
+helm package --version "${RELEASE_VERSION}" "${VCLUSTER_ROOT}/chart" -d "${EMBED_DIR}"

pkg/config/config.go

@@ -22,6 +22,18 @@ type VirtualClusterConfig struct {
 	// Holds the vCluster config
 	config.Config `json:",inline"`
 
+	// WorkloadConfig is the config to access the workload cluster
+	WorkloadConfig *rest.Config `json:"-"`
+
+	// WorkloadClient is the client to access the workload cluster
+	WorkloadClient kubernetes.Interface `json:"-"`
+
+	// ControlPlaneConfig is the config to access the control plane cluster
+	ControlPlaneConfig *rest.Config `json:"-"`
+
+	// ControlPlaneClient is the client to access the control plane cluster
+	ControlPlaneClient kubernetes.Interface `json:"-"`
+
 	// Name is the name of the vCluster
 	Name string `json:"name"`
 
@@ -39,22 +51,10 @@ type VirtualClusterConfig struct {
 
 	// ControlPlaneNamespace is the namespace where the vCluster control plane is running
 	ControlPlaneNamespace string `json:"controlPlaneNamespace,omitempty"`
-
-	// WorkloadConfig is the config to access the workload cluster
-	WorkloadConfig *rest.Config `json:"-"`
-
-	// WorkloadClient is the client to access the workload cluster
-	WorkloadClient kubernetes.Interface `json:"-"`
-
-	// ControlPlaneConfig is the config to access the control plane cluster
-	ControlPlaneConfig *rest.Config `json:"-"`
-
-	// ControlPlaneClient is the client to access the control plane cluster
-	ControlPlaneClient kubernetes.Interface `json:"-"`
 }
 
 func (v VirtualClusterConfig) EmbeddedDatabase() bool {
-	return !v.Config.ControlPlane.BackingStore.Database.External.Enabled && !v.Config.ControlPlane.BackingStore.Etcd.Embedded.Enabled && !v.Config.ControlPlane.BackingStore.Etcd.Deploy.Enabled
+	return !v.ControlPlane.BackingStore.Database.External.Enabled && !v.ControlPlane.BackingStore.Etcd.Embedded.Enabled && !v.ControlPlane.BackingStore.Etcd.Deploy.Enabled
 }
 
 func (v VirtualClusterConfig) VirtualClusterKubeConfig() config.VirtualClusterKubeConfig {
@@ -86,7 +86,7 @@ func (v VirtualClusterConfig) VirtualClusterKubeConfig() config.VirtualClusterKu
 		}
 	}
 
-	retConfig := v.Config.Experimental.VirtualClusterKubeConfig
+	retConfig := v.Experimental.VirtualClusterKubeConfig
 	if retConfig.KubeConfig == "" {
 		retConfig.KubeConfig = distroConfig.KubeConfig
 	}

pkg/config/legacyconfig/config.go

@@ -329,8 +329,10 @@ type Record struct {
 	Namespace  *string    `json:"namespace,omitempty"`
 }
 
-type RecordType string
-type TargetMode string
+type (
+	RecordType string
+	TargetMode string
+)
 
 type Target struct {
 	Mode      TargetMode `json:"mode,omitempty"`

pkg/embed/embed_disabled.go

@@ -1,4 +1,4 @@
-//go:build !embed_charts
+//go:build !embed_chart
 
 package embed
 

pkg/embed/embed_enabled.go

@@ -1,9 +1,9 @@
-//go:build embed_charts
+//go:build embed_chart
 
 package embed
 
 import "embed"
 
-//go:generate ../../hack/embed-charts.sh
-//go:embed charts/*.tgz
+//go:generate ../../hack/embed-chart.sh
+//go:embed chart/*.tgz
 var Charts embed.FS

pkg/helm/secrets.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"sort"
 
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -170,6 +171,48 @@ func (secrets *Secrets) Update(ctx context.Context, secret *corev1.Secret) (*cor
 	return secrets.kubeClient.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{})
 }
 
+// ListUnfiltered fetches all releases and returns the list releases such
+// that filter(release) == true. An error is returned if the
+// secret fails to retrieve the releases.
+func (secrets *Secrets) ListUnfiltered(ctx context.Context, labels kblabels.Selector, namespace string) ([]*Release, error) {
+	req, err := kblabels.NewRequirement("owner", selection.Equals, []string{"helm"})
+	if err != nil {
+		return nil, err
+	}
+	if labels == nil {
+		labels = kblabels.Everything()
+	}
+	labels = labels.Add(*req)
+	list, err := secrets.kubeClient.CoreV1().Secrets(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: labels.String(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// iterate over the secrets object list
+	// and decode each release
+	var releases []*Release
+	for _, item := range list.Items {
+		cpy := item
+		release, err := decodeRelease(&cpy, string(item.Data["release"]))
+		if err != nil {
+			klog.FromContext(ctx).Error(err, "list: failed to decode release")
+			continue
+		} else if release.Chart == nil || release.Chart.Metadata == nil || release.Info == nil {
+			klog.FromContext(ctx).Info("list: metadata info is empty for release", "name", release.Name)
+			continue
+		}
+
+		releases = append(releases, release)
+	}
+
+	sort.Slice(releases, func(i, j int) bool {
+		return releases[i].Version < releases[j].Version
+	})
+	return releases, nil
+}
+
 // List fetches all releases and returns the list releases such
 // that filter(release) == true. An error is returned if the
 // secret fails to retrieve the releases.