feat: common script source for git ops (#890)

* feat: script source

* fix: implementing changes

* fix: tests fixes for new content

* added content library

* fix: reorganized executor library - flattened

* fix: added v2 scripts client

* fix: make fields consistent with testkube api spec

* fix: removed unused func

* fix: fetcher - correct use of tmpFile

* fix: fetcher - dont use dot for full checkout

* fix: comment typo

* operator version bump

* fix: fetcher test as integartion test

* fix: added git to integration tests github action

* fix: test with dot

* fix: removed comment

* fix: removed taras from codeowners

* feat: added paramsFile to Executions resource

* fix: added error handling for nils in fetch

* feat: added content errors

* feat: added content errors

* feat: added paramsFile to script executions

* fix: getting file content

* fix: new string script content func

* fix: script contenttype detect when creating script

* fix: detecting script content

* fix: more verbose minio errors

* fix: debugging

* fix: added debug to job executor

* fix: fixed file name for minio

* fix: tesktube fixed libs for executors

* fix: fixed server.go after merging

.github/CODEOWNERS

@@ -4,4 +4,4 @@ img/* @olensmar
 api/* @olensmar 
 api/v1/* @olensmar 
 
-* @exu @nicufk @jasmingacic @fog1985 @vsukhin
+* @exu @nicufk @jasmingacic @vsukhin

.github/workflows/test.yml

@@ -27,7 +27,9 @@ jobs:
         run: go test -v ./...
 
       - name: Integration tests
-        run: go test --tags=integration -v ./...
+        run: |
+          sudo apt-get install -y git
+          go test --tags=integration -v ./...
 
       # Don't work yet as expected https://github.com/nwestfall/openapi-action/issues/3
       - name: OpenAPI Lint Checks

Makefile

@@ -56,13 +56,13 @@ openapi-generate-model-testkube:
 
 
 test: 
-	go test ./... -cover -v 
+	go test ./... -cover -failfast
 
 test-e2e:
 	go test --tags=e2e -v ./test/e2e
 
 test-integration:
-	go test --tags=integration -v ./...
+	go test -failfast --tags=integration -v ./...
 
 
 test-e2e-namespace:

api/v1/testkube.yaml

@@ -1088,13 +1088,14 @@ components:
         name:
           type: string
           description: script name
+        namespace:
+          type: string
+          description: script namespace
         type:
           type: string
           description: script type
-          enum:
-            - postman/collection
         content:
-          type: string
+          $ref: "#/components/schemas/ScriptContent"
           description: script content
         created:
           type: string
@@ -1105,6 +1106,26 @@ components:
             type: string
           description: script tags
 
+    ScriptContent:
+      type: object
+      properties:
+        type:
+          type: string
+          description: script type
+          enum:
+            - string
+            - file-uri
+            - git-file
+            - git-dir
+        repository:
+          $ref: "#/components/schemas/Repository"
+        data:
+          type: string
+          description: script content data as string
+        uri:
+          type: string
+          description: script content
+
     Execution:
       type: object
       description: API server script execution
@@ -1116,6 +1137,9 @@ components:
         scriptName:
           type: string
           description: unique script name (CRD Script name)
+        scriptNamespace:
+          type: string
+          description: script namespace
         scriptType:
           type: string
           description: script type e.g. postman/collection
@@ -1144,11 +1168,11 @@ components:
           example:
             users: "3"
             prefix: "some-"
-        scriptContent:
+        paramsFile:
           type: string
-          description: script metadata content
-        repository:
-          $ref: "#/components/schemas/Repository"
+          description: params file content - need to be in format for particular executor (e.g. postman envs file)
+        content:
+          $ref: "#/components/schemas/ScriptContent"
         startTime:
           type: string
           description: "test start time"
@@ -1369,19 +1393,7 @@ components:
             users: "3"
             prefix: "some-"
         content:
-          type: string
-          description: script content as string (content depends from executor)
-        inputType:
-          type: string
-          description: >
-            script content type can be: 
-            - direct content - created from file,
-            - git repository with path, will be checked out, useful when test have more than one file or complicated directory structure,
-          enum:
-            - content
-            - git
-        repository:
-          $ref: "#/components/schemas/Repository"
+          $ref: "#/components/schemas/ScriptContent"
 
     Repository:
       description: repository representation for tests in git repositories
@@ -1460,55 +1472,8 @@ components:
 
     ScriptUpsertRequest:
       description: scripts create request body
-      type: object
-      properties:
-        name:
-          type: string
-          description: script name - Custom Resource name - must be unique, use only lowercase numbers and dashes (-)
-          example: kubeshop-homepage-test
-        type:
-          type: string
-          description: script type - what executor type should be used during test execution
-          example: postman/collection
-        namespace:
-          type: string
-          description: kubernetes namespace (defaults to 'testkube')
-          example: testkube
-        inputType:
-          type: string
-          description: >
-            script content type can be: 
-            - direct content - created from file, 
-            - git repo directory checkout in case when test is some kind of project or have more than one file,
-          enum:
-            - content
-            - git
-        tags:
-          type: array
-          items:
-            type: string
-        repository:
-          $ref: "#/components/schemas/Repository"
-        content:
-          type: string
-          description: script content - executor specific e.g. fo postman-collections executor
-          example: >
-            {
-              "info": {
-                "_postman_id": "57ad6291-5b8f-4b2d-b24d-d2d2ce8785bb",
-                "name": "SimpleKubeshop",
-                "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
-              },
-              "item": [
-                {
-                  "name": "Homepage",
-                  "request": {
-                    "method": "GET",
-                    "header": [],
-                    "url": null
-                  },
-                  "response": []
-                }
+      allOf:
+        - $ref: "#/components/schemas/Script"
 
     TestUpsertRequest:
       description: test create request body

cmd/api-server/main.go

@@ -7,9 +7,9 @@ import (
 
 	"github.com/kelseyhightower/envconfig"
 	kubeclient "github.com/kubeshop/testkube-operator/client"
-	executorsclient "github.com/kubeshop/testkube-operator/client/executors"
-	scriptsclient "github.com/kubeshop/testkube-operator/client/scripts"
-	testsclient "github.com/kubeshop/testkube-operator/client/tests"
+	executorsclientv1 "github.com/kubeshop/testkube-operator/client/executors"
+	scriptsclientv2 "github.com/kubeshop/testkube-operator/client/scripts/v2"
+	testsclientv1 "github.com/kubeshop/testkube-operator/client/tests"
 	apiv1 "github.com/kubeshop/testkube/internal/app/api/v1"
 	"github.com/kubeshop/testkube/internal/pkg/api/repository/result"
 	"github.com/kubeshop/testkube/internal/pkg/api/repository/storage"
@@ -55,9 +55,9 @@ func main() {
 	secretClient, err := secret.NewClient()
 	ui.ExitOnError("Getting secret client", err)
 
-	scriptsClient := scriptsclient.NewClient(kubeClient)
-	executorsClient := executorsclient.NewClient(kubeClient)
-	testsClient := testsclient.NewClient(kubeClient)
+	scriptsClient := scriptsclientv2.NewClient(kubeClient)
+	executorsClient := executorsclientv1.NewClient(kubeClient)
+	testsClient := testsclientv1.NewClient(kubeClient)
 
 	resultsRepository := result.NewMongoRespository(db)
 	testResultsRepository := testresult.NewMongoRespository(db)

cmd/kubectl-testkube/commands/crds/scripts_crds.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/kubeshop/testkube/cmd/kubectl-testkube/commands/common/validator"
 	"github.com/kubeshop/testkube/pkg/api/v1/client"
+	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/test/script/detector"
 	"github.com/kubeshop/testkube/pkg/ui"
 	"github.com/spf13/cobra"
@@ -92,7 +93,7 @@ spec:
 
 	// try to detect type if none passed
 	d := detector.NewDefaultDetector()
-	if detectedType, ok := d.Detect(client.UpsertScriptOptions{Content: string(content)}); ok {
+	if detectedType, ok := d.Detect(client.UpsertScriptOptions{Content: &testkube.ScriptContent{Data: string(content)}}); ok {
 		ui.Debug("Detected test script type", detectedType)
 		scriptType = detectedType
 	} else {

cmd/kubectl-testkube/commands/executors/create.go

@@ -14,9 +14,10 @@ func NewCreateExecutorCmd() *cobra.Command {
 	)
 
 	cmd := &cobra.Command{
-		Use:   "create",
-		Short: "Create new Executor",
-		Long:  `Create new Executor Custom Resource`,
+		Use:     "create",
+		Aliases: []string{"c"},
+		Short:   "Create new Executor",
+		Long:    `Create new Executor Custom Resource`,
 		Run: func(cmd *cobra.Command, args []string) {
 			ui.Logo()
 

cmd/kubectl-testkube/commands/scripts/common.go

@@ -2,13 +2,17 @@ package scripts
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"reflect"
 	"time"
 
-	"github.com/kubeshop/testkube/pkg/api/v1/client"
+	apiclientv1 "github.com/kubeshop/testkube/pkg/api/v1/client"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
-	"github.com/kubeshop/testkube/pkg/runner/output"
+	"github.com/kubeshop/testkube/pkg/executor/output"
+	"github.com/kubeshop/testkube/pkg/test/script/detector"
 	"github.com/kubeshop/testkube/pkg/ui"
+	"github.com/spf13/cobra"
 )
 
 func printExecutionDetails(execution testkube.Execution) {
@@ -20,7 +24,7 @@ func printExecutionDetails(execution testkube.Execution) {
 	ui.NL()
 }
 
-func DownloadArtifacts(id, dir string, client client.Client) {
+func DownloadArtifacts(id, dir string, client apiclientv1.Client) {
 	artifacts, err := client.GetExecutionArtifacts(id)
 	ui.ExitOnError("getting artifacts ", err)
 
@@ -40,7 +44,7 @@ func DownloadArtifacts(id, dir string, client client.Client) {
 	ui.NL()
 }
 
-func watchLogs(id string, client client.Client) {
+func watchLogs(id string, client apiclientv1.Client) {
 	ui.Info("Getting pod logs")
 
 	logs, err := client.Logs(id)
@@ -84,3 +88,117 @@ func watchLogs(id string, client client.Client) {
 
 	uiShellGetExecution(id)
 }
+
+func newContentFromFlags(cmd *cobra.Command) (content *testkube.ScriptContent, err error) {
+	var fileContent []byte
+
+	scriptContentType := cmd.Flag("script-content-type").Value.String()
+	file := cmd.Flag("file").Value.String()
+	uri := cmd.Flag("uri").Value.String()
+	gitUri := cmd.Flag("git-uri").Value.String()
+	gitBranch := cmd.Flag("git-branch").Value.String()
+	gitPath := cmd.Flag("git-path").Value.String()
+	gitUsername := cmd.Flag("git-username").Value.String()
+	gitToken := cmd.Flag("git-token").Value.String()
+
+	// get file content
+	if file != "" {
+		fileContent, err = ioutil.ReadFile(file)
+		if err != nil {
+			return content, fmt.Errorf("reading file "+file+" error: %w", err)
+		}
+	} else if stat, _ := os.Stdin.Stat(); (stat.Mode() & os.ModeCharDevice) == 0 {
+		fileContent, err = ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			return content, fmt.Errorf("reading stdin error: %w", err)
+		}
+	}
+
+	// content is correct when is passed from file, by uri, ur by git repo
+	if len(fileContent) == 0 && uri == "" && gitUri == "" {
+		return content, fmt.Errorf("empty script content, please pass some script content to create script")
+	}
+
+	// detect content type (git-file need to be everrided manually as we don't)
+	// TODO handle git-file somehow
+	if gitUri != "" && scriptContentType == "" {
+		scriptContentType = string(testkube.ScriptContentTypeGitDir)
+	}
+
+	if uri != "" && scriptContentType == "" {
+		scriptContentType = string(testkube.ScriptContentTypeFileURI)
+	}
+
+	if len(fileContent) > 0 {
+		scriptContentType = string(testkube.ScriptContentTypeString)
+	}
+
+	var repository *testkube.Repository
+	if gitUri != "" && gitBranch != "" {
+		if scriptContentType == "" {
+			scriptContentType = "git-dir"
+		}
+
+		repository = &testkube.Repository{
+			Type_:    "git",
+			Uri:      gitUri,
+			Branch:   gitBranch,
+			Path:     gitPath,
+			Username: gitUsername,
+			Token:    gitToken,
+		}
+	}
+
+	content = &testkube.ScriptContent{
+		Type_:      scriptContentType,
+		Data:       string(fileContent),
+		Repository: repository,
+		Uri:        uri,
+	}
+
+	return content, nil
+}
+
+func NewUpsertScriptOptionsFromFlags(cmd *cobra.Command, script testkube.Script) (options apiclientv1.UpsertScriptOptions, err error) {
+	content, err := newContentFromFlags(cmd)
+
+	ui.ExitOnError("creating content from passed parameters", err)
+
+	name := cmd.Flag("name").Value.String()
+	executorType := cmd.Flag("type").Value.String()
+	namespace := cmd.Flag("script-namespace").Value.String()
+	tags, err := cmd.Flags().GetStringSlice("tags")
+	if err != nil {
+		return options, err
+	}
+
+	options = apiclientv1.UpsertScriptOptions{
+		Name:      name,
+		Type_:     executorType,
+		Content:   content,
+		Namespace: namespace,
+	}
+
+	// if tags are passed and are different from the existing overwrite
+	if len(tags) > 0 && !reflect.DeepEqual(script.Tags, tags) {
+		options.Tags = tags
+	} else {
+		options.Tags = script.Tags
+	}
+
+	// try to detect type if none passed
+	if executorType == "" {
+		d := detector.NewDefaultDetector()
+		if detectedType, ok := d.Detect(options); ok {
+			ui.Info("Detected test script type", detectedType)
+			options.Type_ = detectedType
+		}
+	}
+
+	if options.Type_ == "" {
+		return options, fmt.Errorf("can't detect executor type by passed file content (%s), please pass valid --type flag", executorType)
+	}
+
+	return options, nil
+
+}