fix(TKC-3318): auto-secrets creation (#6208)

* feat: add option to disable cloud storage in DevBox * fix: auto-create secrets correctly - make ownership UID valid
kubeshop · Feb 28, 2025 · f40a951 · f40a951
1 parent 2bd61a3
commit f40a951
Show file tree

Hide file tree

Showing 13 changed files with 161 additions and 54 deletions.
diff --git a/cmd/tcl/kubectl-testkube/devbox/command.go b/cmd/tcl/kubectl-testkube/devbox/command.go
@@ -55,6 +55,7 @@ func NewDevBoxCommand() *cobra.Command {
 		runnersCount        uint16
 		gitopsEnabled       bool
 		disableDefaultAgent bool
+		disableCloudStorage bool
 	)
 
 	cmd := &cobra.Command{
@@ -144,7 +145,7 @@ func NewDevBoxCommand() *cobra.Command {
 
 			// Initialize wrappers over cluster resources
 			interceptor := devutils.NewInterceptor(interceptorPod, baseInitImage, baseToolkitImage, interceptorBin)
-			agent := devutils.NewAgent(agentPod, cloud, baseAgentImage, baseInitImage, baseToolkitImage)
+			agent := devutils.NewAgent(agentPod, cloud, baseAgentImage, baseInitImage, baseToolkitImage, disableCloudStorage)
 			binaryStorage := devutils.NewBinaryStorage(binaryStoragePod, binaryStorageBin)
 			mongo := devutils.NewMongo(mongoPod)
 			minio := devutils.NewMinio(minioPod)
@@ -190,7 +191,7 @@ func NewDevBoxCommand() *cobra.Command {
 			// Create environment in the Cloud
 			if !oss {
 				fmt.Println("Creating environment in Cloud...")
-				env, err = cloud.CreateEnvironment(namespace.Name())
+				env, err = cloud.CreateEnvironment(namespace.Name(), disableCloudStorage)
 				if err != nil {
 					fail(errors.Wrap(err, "failed to create Cloud environment"))
 				}
@@ -874,6 +875,7 @@ func NewDevBoxCommand() *cobra.Command {
 	cmd.Flags().StringVar(&baseAgentImage, "agent-image", "kubeshop/testkube-api-server:latest", "base agent image")
 	cmd.Flags().Uint16Var(&runnersCount, "runners", 0, "additional runners count")
 	cmd.Flags().BoolVar(&disableDefaultAgent, "disable-agent", false, "should disable default agent")
+	cmd.Flags().BoolVar(&disableCloudStorage, "disable-cloud-storage", false, "should disable storage in Cloud")
 
 	return cmd
 }
diff --git a/cmd/tcl/kubectl-testkube/devbox/devutils/agent.go b/cmd/tcl/kubectl-testkube/devbox/devutils/agent.go
@@ -10,6 +10,7 @@ package devutils
 
 import (
 	"context"
+	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -19,20 +20,22 @@ import (
 )
 
 type Agent struct {
-	pod              *PodObject
-	cloud            *CloudObject
-	agentImage       string
-	initProcessImage string
-	toolkitImage     string
+	pod                 *PodObject
+	cloud               *CloudObject
+	agentImage          string
+	initProcessImage    string
+	toolkitImage        string
+	disableCloudStorage bool
 }
 
-func NewAgent(pod *PodObject, cloud *CloudObject, agentImage, initProcessImage, toolkitImage string) *Agent {
+func NewAgent(pod *PodObject, cloud *CloudObject, agentImage, initProcessImage, toolkitImage string, disableCloudStorage bool) *Agent {
 	return &Agent{
-		pod:              pod,
-		cloud:            cloud,
-		agentImage:       agentImage,
-		initProcessImage: initProcessImage,
-		toolkitImage:     toolkitImage,
+		pod:                 pod,
+		cloud:               cloud,
+		agentImage:          agentImage,
+		initProcessImage:    initProcessImage,
+		toolkitImage:        toolkitImage,
+		disableCloudStorage: disableCloudStorage,
 	}
 }
 
@@ -53,7 +56,7 @@ func (r *Agent) Create(ctx context.Context, env *client.Environment) error {
 		{Name: "TESTKUBE_TW_TOOLKIT_IMAGE", Value: r.toolkitImage},
 		{Name: "TESTKUBE_TW_INIT_IMAGE", Value: r.initProcessImage},
 		{Name: "FEATURE_NEW_ARCHITECTURE", Value: "true"},
-		{Name: "FEATURE_CLOUD_STORAGE", Value: "true"},
+		{Name: "FEATURE_CLOUD_STORAGE", Value: fmt.Sprintf("%v", !r.disableCloudStorage)},
 	}
 	if env != nil {
 		tlsInsecure := "false"

diff --git a/cmd/tcl/kubectl-testkube/devbox/devutils/cloud.go b/cmd/tcl/kubectl-testkube/devbox/devutils/cloud.go
@@ -148,12 +148,12 @@ func (c *CloudObject) DashboardUrl(id, path string) string {
 	return strings.TrimSuffix(fmt.Sprintf("%s/organization/%s/environment/%s/", c.cfg.UiUri, c.cfg.OrganizationId, id)+strings.TrimPrefix(path, "/"), "/")
 }
 
-func (c *CloudObject) CreateEnvironment(name string) (*client.Environment, error) {
+func (c *CloudObject) CreateEnvironment(name string, disableCloudStorage bool) (*client.Environment, error) {
 	env, err := c.envClient.Create(client.Environment{
 		Name:            name,
 		Owner:           c.cfg.OrganizationId,
 		OrganizationId:  c.cfg.OrganizationId,
-		CloudStorage:    true,
+		CloudStorage:    !disableCloudStorage,
 		NewArchitecture: true,
 	})
 	if err != nil {

diff --git a/internal/app/api/v1/testworkflows.go b/internal/app/api/v1/testworkflows.go
@@ -203,17 +203,24 @@ func (s *TestkubeAPI) CreateTestWorkflowHandler() fiber.Handler {
 		}
 
 		// Create secrets
-		err = s.SecretManager.InsertBatch(ctx, execNamespace, secrets, &metav1.OwnerReference{
-			APIVersion: testworkflowsv1.GroupVersion.String(),
-			Kind:       testworkflowsv1.Resource,
-			Name:       obj.Name,
-			UID:        obj.UID,
-		})
-		s.Metrics.IncCreateTestWorkflow(err)
-		if err != nil {
-			_ = s.TestWorkflowsClient.Delete(context.Background(), environmentId, obj.Name)
-			return s.BadRequest(c, errPrefix, "auto-creating secrets", err)
+		if secrets.HasData() {
+			uid, _ := s.TestWorkflowsClient.GetKubernetesObjectUID(ctx, environmentId, obj.Name)
+			var ref *metav1.OwnerReference
+			if uid != "" {
+				ref = &metav1.OwnerReference{
+					APIVersion: testworkflowsv1.GroupVersion.String(),
+					Kind:       testworkflowsv1.Resource,
+					Name:       obj.Name,
+					UID:        uid,
+				}
+			}
+			err = s.SecretManager.InsertBatch(ctx, execNamespace, secrets, ref)
+			if err != nil {
+				_ = s.TestWorkflowsClient.Delete(context.Background(), environmentId, obj.Name)
+				return s.BadRequest(c, errPrefix, "auto-creating secrets", err)
+			}
 		}
+		s.Metrics.IncCreateTestWorkflow(err)
 
 		s.sendCreateWorkflowTelemetry(ctx, obj)
 
@@ -284,21 +291,28 @@ func (s *TestkubeAPI) UpdateTestWorkflowHandler() fiber.Handler {
 			return s.BadRequest(c, errPrefix, "client error", err)
 		}
 
-		// Create secrets
-		err = s.SecretManager.InsertBatch(c.Context(), execNamespace, secrets, &metav1.OwnerReference{
-			APIVersion: testworkflowsv1.GroupVersion.String(),
-			Kind:       testworkflowsv1.Resource,
-			Name:       obj.Name,
-			UID:        obj.UID,
-		})
-		s.Metrics.IncUpdateTestWorkflow(err)
-		if err != nil {
-			err = s.TestWorkflowsClient.Update(context.Background(), environmentId, *initial)
+		// Create secrets if necessary
+		if secrets.HasData() {
+			uid, _ := s.TestWorkflowsClient.GetKubernetesObjectUID(ctx, environmentId, obj.Name)
+			var ref *metav1.OwnerReference
+			if uid != "" {
+				ref = &metav1.OwnerReference{
+					APIVersion: testworkflowsv1.GroupVersion.String(),
+					Kind:       testworkflowsv1.Resource,
+					Name:       obj.Name,
+					UID:        uid,
+				}
+			}
+			err = s.SecretManager.InsertBatch(c.Context(), execNamespace, secrets, ref)
 			if err != nil {
-				s.Log.Errorf("failed to recover previous TestWorkflow state: %v", err)
+				err = s.TestWorkflowsClient.Update(context.Background(), environmentId, *initial)
+				if err != nil {
+					s.Log.Errorf("failed to recover previous TestWorkflow state: %v", err)
+				}
+				return s.BadRequest(c, errPrefix, "auto-creating secrets", err)
 			}
-			return s.BadRequest(c, errPrefix, "auto-creating secrets", err)
 		}
+		s.Metrics.IncUpdateTestWorkflow(err)
 
 		err = SendResource(c, "TestWorkflow", testworkflowsv1.GroupVersion, testworkflows.MapKubeToAPI, obj)
 		if err != nil {

diff --git a/internal/app/api/v1/testworkflowtemplates.go b/internal/app/api/v1/testworkflowtemplates.go
@@ -145,17 +145,24 @@ func (s *TestkubeAPI) CreateTestWorkflowTemplateHandler() fiber.Handler {
 		}
 
 		// Create secrets
-		err = s.SecretManager.InsertBatch(c.Context(), execNamespace, secrets, &metav1.OwnerReference{
-			APIVersion: testworkflowsv1.GroupVersion.String(),
-			Kind:       testworkflowsv1.ResourceTemplate,
-			Name:       obj.Name,
-			UID:        obj.UID,
-		})
-		s.Metrics.IncCreateTestWorkflowTemplate(err)
-		if err != nil {
-			_ = s.TestWorkflowTemplatesClient.Delete(ctx, environmentId, obj.Name)
-			return s.BadRequest(c, errPrefix, "auto-creating secrets", err)
+		if secrets.HasData() {
+			uid, _ := s.TestWorkflowTemplatesClient.GetKubernetesObjectUID(ctx, environmentId, obj.Name)
+			var ref *metav1.OwnerReference
+			if uid != "" {
+				ref = &metav1.OwnerReference{
+					APIVersion: testworkflowsv1.GroupVersion.String(),
+					Kind:       testworkflowsv1.ResourceTemplate,
+					Name:       obj.Name,
+					UID:        uid,
+				}
+			}
+			err = s.SecretManager.InsertBatch(c.Context(), execNamespace, secrets, ref)
+			if err != nil {
+				_ = s.TestWorkflowTemplatesClient.Delete(ctx, environmentId, obj.Name)
+				return s.BadRequest(c, errPrefix, "auto-creating secrets", err)
+			}
 		}
+		s.Metrics.IncCreateTestWorkflowTemplate(err)
 		s.sendCreateWorkflowTemplateTelemetry(c.Context(), obj)
 
 		err = SendResource(c, "TestWorkflowTemplate", testworkflowsv1.GroupVersion, testworkflows.MapTemplateKubeToAPI, obj)
@@ -226,12 +233,19 @@ func (s *TestkubeAPI) UpdateTestWorkflowTemplateHandler() fiber.Handler {
 		}
 
 		// Create secrets
-		err = s.SecretManager.InsertBatch(c.Context(), execNamespace, secrets, &metav1.OwnerReference{
-			APIVersion: testworkflowsv1.GroupVersion.String(),
-			Kind:       testworkflowsv1.ResourceTemplate,
-			Name:       obj.Name,
-			UID:        obj.UID,
-		})
+		if secrets.HasData() {
+			uid, _ := s.TestWorkflowTemplatesClient.GetKubernetesObjectUID(ctx, environmentId, obj.Name)
+			var ref *metav1.OwnerReference
+			if uid != "" {
+				ref = &metav1.OwnerReference{
+					APIVersion: testworkflowsv1.GroupVersion.String(),
+					Kind:       testworkflowsv1.ResourceTemplate,
+					Name:       obj.Name,
+					UID:        uid,
+				}
+			}
+			err = s.SecretManager.InsertBatch(c.Context(), execNamespace, secrets, ref)
+		}
 		s.Metrics.IncUpdateTestWorkflowTemplate(err)
 		if err != nil {
 			err = s.TestWorkflowTemplatesClient.Update(ctx, environmentId, *initial)

diff --git a/pkg/newclients/testworkflowclient/cloud.go b/pkg/newclients/testworkflowclient/cloud.go
@@ -3,6 +3,8 @@ package testworkflowclient
 import (
 	"context"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/kubeshop/testkube/internal/common"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/cloud"
@@ -59,6 +61,10 @@ func (c *cloudTestWorkflowClient) DeleteByLabels(ctx context.Context, environmen
 	return c.client.DeleteTestWorkflowsByLabels(ctx, environmentId, labels)
 }
 
+func (c *cloudTestWorkflowClient) GetKubernetesObjectUID(ctx context.Context, environmentId string, name string) (types.UID, error) {
+	return "", nil
+}
+
 func (c *cloudTestWorkflowClient) WatchUpdates(ctx context.Context, environmentId string, includeInitialData bool) Watcher {
 	return channels.Transform(c.client.WatchTestWorkflowUpdates(ctx, environmentId, includeInitialData), func(t *controlplaneclient.TestWorkflowUpdate) (Update, bool) {
 		switch t.Type {

diff --git a/pkg/newclients/testworkflowclient/interface.go b/pkg/newclients/testworkflowclient/interface.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"time"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/repository/channels"
 )
@@ -34,6 +36,7 @@ type Watcher channels.Watcher[Update]
 //go:generate mockgen -destination=./mock_interface.go -package=testworkflowclient "github.com/kubeshop/testkube/pkg/newclients/testworkflowclient" TestWorkflowClient
 type TestWorkflowClient interface {
 	Get(ctx context.Context, environmentId string, name string) (*testkube.TestWorkflow, error)
+	GetKubernetesObjectUID(ctx context.Context, environmentId string, name string) (types.UID, error)
 	List(ctx context.Context, environmentId string, options ListOptions) ([]testkube.TestWorkflow, error)
 	ListLabels(ctx context.Context, environmentId string) (map[string][]string, error)
 	Update(ctx context.Context, environmentId string, workflow testkube.TestWorkflow) error

diff --git a/pkg/newclients/testworkflowclient/kubernetes.go b/pkg/newclients/testworkflowclient/kubernetes.go
@@ -12,6 +12,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -83,6 +84,17 @@ func (c *k8sTestWorkflowClient) Get(ctx context.Context, environmentId string, n
 	return testworkflows.MapKubeToAPI(workflow), nil
 }
 
+func (c *k8sTestWorkflowClient) GetKubernetesObjectUID(ctx context.Context, environmentId string, name string) (types.UID, error) {
+	workflow, err := c.get(ctx, name)
+	if err != nil {
+		return "", err
+	}
+	if workflow != nil {
+		return workflow.UID, nil
+	}
+	return "", nil
+}
+
 func (c *k8sTestWorkflowClient) List(ctx context.Context, environmentId string, options ListOptions) ([]testkube.TestWorkflow, error) {
 	labelSelector := labels2.NewSelector()
 	for k, v := range options.Labels {

diff --git a/pkg/newclients/testworkflowclient/mock_interface.go b/pkg/newclients/testworkflowclient/mock_interface.go
diff --git a/pkg/newclients/testworkflowtemplateclient/cloud.go b/pkg/newclients/testworkflowtemplateclient/cloud.go
@@ -3,6 +3,8 @@ package testworkflowtemplateclient
 import (
 	"context"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/kubeshop/testkube/internal/common"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/cloud"
@@ -24,6 +26,10 @@ func (c *cloudTestWorkflowTemplateClient) Get(ctx context.Context, environmentId
 	return c.client.GetTestWorkflowTemplate(ctx, environmentId, name)
 }
 
+func (c *cloudTestWorkflowTemplateClient) GetKubernetesObjectUID(ctx context.Context, environmentId string, name string) (types.UID, error) {
+	return "", nil
+}
+
 func (c *cloudTestWorkflowTemplateClient) List(ctx context.Context, environmentId string, options ListOptions) ([]testkube.TestWorkflowTemplate, error) {
 	list, err := c.client.ListTestWorkflowTemplates(ctx, environmentId, controlplaneclient.ListTestWorkflowTemplateOptions{
 		Labels:     options.Labels,

diff --git a/pkg/newclients/testworkflowtemplateclient/interface.go b/pkg/newclients/testworkflowtemplateclient/interface.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"time"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/repository/channels"
 )
@@ -34,6 +36,7 @@ type Watcher channels.Watcher[Update]
 //go:generate mockgen -destination=./mock_interface.go -package=testworkflowtemplateclient "github.com/kubeshop/testkube/pkg/newclients/testworkflowtemplateclient" TestWorkflowTemplateClient
 type TestWorkflowTemplateClient interface {
 	Get(ctx context.Context, environmentId string, name string) (*testkube.TestWorkflowTemplate, error)
+	GetKubernetesObjectUID(ctx context.Context, environmentId string, name string) (types.UID, error)
 	List(ctx context.Context, environmentId string, options ListOptions) ([]testkube.TestWorkflowTemplate, error)
 	ListLabels(ctx context.Context, environmentId string) (map[string][]string, error)
 	Update(ctx context.Context, environmentId string, template testkube.TestWorkflowTemplate) error

diff --git a/pkg/newclients/testworkflowtemplateclient/kubernetes.go b/pkg/newclients/testworkflowtemplateclient/kubernetes.go
@@ -12,6 +12,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -83,6 +84,17 @@ func (c *k8sTestWorkflowTemplateClient) Get(ctx context.Context, environmentId s
 	return testworkflows.MapTemplateKubeToAPI(template), nil
 }
 
+func (c *k8sTestWorkflowTemplateClient) GetKubernetesObjectUID(ctx context.Context, environmentId string, name string) (types.UID, error) {
+	template, err := c.get(ctx, name)
+	if err != nil {
+		return "", err
+	}
+	if template != nil {
+		return template.UID, nil
+	}
+	return "", nil
+}
+
 func (c *k8sTestWorkflowTemplateClient) List(ctx context.Context, environmentId string, options ListOptions) ([]testkube.TestWorkflowTemplate, error) {
 	labelSelector := labels2.NewSelector()
 	for k, v := range options.Labels {