fix: validate and generate with sub-chart dep (#46)

Co-authored-by: Giulio <giuliocalzolari@users.noreply.github.com>

.github/workflows/test.yml

@@ -11,10 +11,9 @@ on:
       - "plugin.yaml"
 
 permissions:
-  contents: write
-  pull-requests: write
+  contents: read
+  pull-requests: read
   actions: read
-  checks: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -34,22 +33,7 @@ jobs:
       - name: Set up Helm
         uses: azure/setup-helm@v4
         with:
-          version: v3.12.1
-
-      - name: helm build dependency
-        run: |
-          # This step is needed to make HELM_DATAROBOT_TEST_SKIP_DEPENDENCY_UPDATE=true work in unit tests
-          helm dependency update testdata/test-chart3
-          helm dependency update testdata/test-chart2
-          helm dependency update testdata/test-chart1
-
-      - name: run unit test
-        run: |
-          export GOPROXY=https://proxy.golang.org
-          export HELM_DATAROBOT_TEST_SKIP_DEPENDENCY_UPDATE=true
-          go run gotest.tools/gotestsum@v1.12.0 --format short-verbose --junitfile testResult_unit.xml
-          ls -la .
-          cat testResult_unit.xml
+          version: v3.16.3
 
       - name: Docs
         run: |
@@ -60,11 +44,6 @@ jobs:
             exit 1
           fi
 
-      - name: Report
-        uses: dorny/test-reporter@v1
-        if: always()
-        with:
-          name: unit Tests
-          path: "testResult_unit.xml"
-          reporter: java-junit
-          fail-on-error: true
+      - name: run unit test
+        run: |
+          make test

Makefile

@@ -15,7 +15,13 @@ docs: build
 	@mkdir ./docs
 	@./$(APP_NAME) docs --path ./docs
 
-test:
+pre-test:
+	@echo "Pre tests..."
+	@helm dependency update testdata/test-chart3
+	@helm dependency update testdata/test-chart2
+	@helm dependency update testdata/test-chart1
+
+test: pre-test
 	@echo "Running tests..."
 	$(GO) test ./... -v
 

cmd/generate.go

@@ -2,17 +2,16 @@ package cmd
 
 import (
 	"fmt"
-	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
+	"github.com/datarobot-oss/helm-datarobot-plugin/pkg/chartutil"
 	"github.com/datarobot-oss/helm-datarobot-plugin/pkg/image_uri"
+	"github.com/datarobot-oss/helm-datarobot-plugin/pkg/render_helper"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
-	"helm.sh/helm/v3/pkg/chart/loader"
-	"helm.sh/helm/v3/pkg/chartutil"
-	"helm.sh/helm/v3/pkg/engine"
 )
 
 var generateCmd = &cobra.Command{
@@ -32,55 +31,28 @@ $ helm datarobot generate chart.tgz
 	Args: cobra.MinimumNArgs(1), // Requires at least one argument (file path)
 	RunE: func(cmd *cobra.Command, args []string) error {
 		chartPath := args[0]
-		chart, err := loader.Load(chartPath)
+		manifest, err := render_helper.NewRenderItems(chartPath)
 		if err != nil {
 			return fmt.Errorf("Error loading chart %s: %v", chartPath, err)
 		}
 
-		options := chartutil.ReleaseOptions{
-			Name:      "test-release",
-			Namespace: "test",
-		}
-		values := map[string]interface{}{}
-		caps := chartutil.DefaultCapabilities.Copy()
-
-		cvals, err := chartutil.CoalesceValues(chart, values)
-		if err != nil {
-			return fmt.Errorf("Error CoalesceValues chart %s: %v", chartPath, err)
-		}
-
-		valuesToRender, err := chartutil.ToRenderValuesWithSchemaValidation(chart, cvals, options, caps, true)
-		if err != nil {
-			return fmt.Errorf("Error ToRenderValuesWithSchemaValidation chart %s: %v", chartPath, err)
-		}
-		var e engine.Engine
-
-		renderedContentMap, err := e.Render(chart, valuesToRender)
-		if err != nil {
-			return fmt.Errorf("Error Render chart %s: %v", chartPath, err)
-		}
-
-		var sb strings.Builder
-		uniqueEntries := make(map[string]struct{})
-		for _, template := range chart.Templates {
-			fileName, _ := template.Name, template.Data
-			// We only apply the following lint rules to yaml files
+		uniqueEntries := make(map[string]string)
+		for fileName, template := range manifest {
+			// // We only apply the following lint rules to yaml files
 			if filepath.Ext(fileName) != ".yaml" || filepath.Ext(fileName) == ".yml" {
 				continue
 			}
 
-			renderedContent := renderedContentMap[path.Join(chart.Name(), fileName)]
 			if generateDebug {
-				fmt.Printf("---\n# Source: %s\n%s\n", fileName, renderedContent)
+				fmt.Printf("---\n# Source: %s\n%s\n", fileName, template)
 			}
 
-			manifestImages, err := ExtractImagesFromManifest(renderedContent)
+			manifestImages, err := ExtractImagesFromManifest(template)
 			if err != nil {
-				return fmt.Errorf("Error ExtractImagesFromManifest chart %s: %v", chartPath, err)
+				return fmt.Errorf("Error ExtractImagesFromManifest chart %s: %v", fileName, err)
 			}
 
 			re := regexp.MustCompile("[^a-zA-Z0-9]+")
-
 			for _, item := range manifestImages {
 				iUri, err := image_uri.NewDockerUri(item)
 				if err != nil {
@@ -90,17 +62,34 @@ $ helm datarobot generate chart.tgz
 				// Check if the item is already in the map
 				if _, exists := uniqueEntries[uniqueKey]; !exists {
 					// If not, add it to the map and the finalSlice
-					uniqueEntries[uniqueKey] = struct{}{}
-					sb.WriteString(fmt.Sprintf("- name: %s\n", uniqueKey))
-					sb.WriteString(fmt.Sprintf("  image: %s\n", item))
+					uniqueEntries[uniqueKey] = item
 				}
 			}
 
 		}
 
+		var keys []string
+		for key := range uniqueEntries {
+			keys = append(keys, key)
+		}
+
+		// Sort the keys
+		sort.Strings(keys)
+
+		// Create a slice to hold the items
+		var items []chartutil.DatarobotImageDeclaration
+		for _, key := range keys {
+			items = append(items, chartutil.DatarobotImageDeclaration{Name: key, Image: uniqueEntries[key]})
+		}
+
+		yamlItems, err := yaml.Marshal(items)
+		if err != nil {
+			return fmt.Errorf("Error converting to YAML: %v\n", err)
+		}
+
 		output := map[string]interface{}{
 			"annotations": map[string]string{
-				string(annotation): sb.String(),
+				string(annotation): string(yamlItems),
 			},
 		}
 

cmd/generate_test.go

@@ -8,7 +8,7 @@ import (
 
 func TestCommandGenerate(t *testing.T) {
 	t.Run("test-chart5", func(t *testing.T) {
-		output, err := executeCommand(rootCmd, "generate", "../testdata/test-chart5")
+		output, err := executeCommand(rootCmd, "generate ../testdata/test-chart5")
 		assert.NoError(t, err)
 		expectedOutput := `annotations:
   datarobot.com/images: |
@@ -18,4 +18,19 @@ func TestCommandGenerate(t *testing.T) {
       image: docker.io/alpine/curl:8.9.1`
 		assert.Equal(t, expectedOutput, output)
 	})
+
+	t.Run("test-chart1", func(t *testing.T) {
+		output, err := executeCommand(rootCmd, "generate ../testdata/test-chart1")
+		assert.NoError(t, err)
+		expectedOutput := `annotations:
+  datarobot.com/images: |
+    - name: test-image1_100
+      image: docker.io/datarobotdev/test-image1:1.0.0
+    - name: test-image2_200
+      image: docker.io/datarobotdev/test-image2:2.0.0
+    - name: test-image3_300
+      image: docker.io/datarobotdev/test-image3:3.0.0`
+		assert.Equal(t, expectedOutput, output)
+	})
+
 }

cmd/images_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestCommandImages(t *testing.T) {
-	output, err := executeCommand(rootCmd, "image", "../testdata/test-chart1")
+	output, err := executeCommand(rootCmd, "image ../testdata/test-chart1")
 	assert.NoError(t, err)
 	expectedOutput := `- name: test-image1
   image: docker.io/datarobotdev/test-image1:1.0.0

cmd/load_test.go

@@ -11,7 +11,7 @@ func TestCommandLoad(t *testing.T) {
 
 	t.Run("test-chart", func(t *testing.T) {
 		filePath := "image-load.tgz"
-		output, err := executeCommand(rootCmd, "save", "../testdata/test-chart4", "-a", "custom/loadimages", "--output", filePath)
+		output, err := executeCommand(rootCmd, "save ../testdata/test-chart4 -a custom/loadimages --output "+filePath)
 		assert.NoError(t, err)
 		expectedSaveOutput := `Pulling image: docker.io/alpine/curl:8.9.1
 Pulling image: docker.io/busybox:1.36.1
@@ -23,7 +23,7 @@ Tarball created successfully: image-load.tgz`
 			t.Errorf("File was not created: %s", filePath)
 		}
 
-		output, err = executeCommand(rootCmd, "load", filePath, "-r", "ttl.sh")
+		output, err = executeCommand(rootCmd, "load "+filePath+" -r ttl.sh")
 		assert.NoError(t, err)
 		expectedLoadOutput := `Successfully pushed image ttl.sh/curl:8.9.1
 Successfully pushed image ttl.sh/busybox:1.36.1`

cmd/release_manifest_test.go

@@ -8,7 +8,7 @@ import (
 
 func TestCommandReleaseManifest(t *testing.T) {
 	t.Run("Test test-chart1", func(t *testing.T) {
-		output, err := executeCommand(rootCmd, "release-manifest", "../testdata/test-chart1", "-a", "datarobot.com/images")
+		output, err := executeCommand(rootCmd, "release-manifest ../testdata/test-chart1 -a \"datarobot.com/images\"")
 		assert.NoError(t, err)
 		// Expected output to compare
 		expectedOutput := `images:
@@ -29,7 +29,7 @@ func TestCommandReleaseManifest(t *testing.T) {
 	})
 
 	t.Run("Test test-chart4", func(t *testing.T) {
-		output, err := executeCommand(rootCmd, "release-manifest", "../testdata/test-chart4", "-a", "custom/images")
+		output, err := executeCommand(rootCmd, "release-manifest ../testdata/test-chart4 -a \"custom/images\"")
 		assert.NoError(t, err)
 		expectedOutput := `images:
   test-image4.tar.zst:
@@ -40,7 +40,7 @@ func TestCommandReleaseManifest(t *testing.T) {
 	})
 
 	t.Run("Test test-chart4-datarobot", func(t *testing.T) {
-		output, err := executeCommand(rootCmd, "release-manifest", "../testdata/test-chart4", "-a", "datarobot.com/images")
+		output, err := executeCommand(rootCmd, "release-manifest ../testdata/test-chart4 -a \"datarobot.com/images\"")
 		assert.NoError(t, err)
 		expectedOutput := `images:
   test-image3.tar.zst:
@@ -60,7 +60,7 @@ func TestCommandReleaseManifest(t *testing.T) {
 		assert.Equal(t, expectedOutput, output)
 	})
 	t.Run("Test test-chart4-duplicated", func(t *testing.T) {
-		output, err := executeCommand(rootCmd, "release-manifest", "../testdata/test-chart4", "-a", "custom/images-duplicated")
+		output, err := executeCommand(rootCmd, "release-manifest ../testdata/test-chart4 -a \"custom/images-duplicated\"")
 		assert.NoError(t, err)
 		// Expected output to compare
 		expectedOutput := `images:

cmd/root.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/mattn/go-shellwords"
 	"github.com/spf13/cobra"
 )
 
@@ -36,8 +37,14 @@ func Execute() {
 	}
 }
 
-func executeCommand(root *cobra.Command, args ...string) (output string, err error) {
+func executeCommand(root *cobra.Command, cmd string) (output string, err error) {
 	buf := new(bytes.Buffer)
+
+	args, err := shellwords.Parse(cmd)
+	if err != nil {
+		return "", err
+	}
+
 	root.SetOut(buf)
 	root.SetErr(buf)
 	root.SetArgs(args)

cmd/save_test.go

@@ -11,7 +11,7 @@ func TestCommandSave(t *testing.T) {
 
 	t.Run("test-chart4-dry-run", func(t *testing.T) {
 
-		output, err := executeCommand(rootCmd, "save", "../testdata/test-chart4", "-a", "custom/images-duplicated", "--dry-run", "--output", "test.tgz", "-a", "datarobot.com/images")
+		output, err := executeCommand(rootCmd, "save ../testdata/test-chart4 -a custom/images-duplicated --dry-run --output test.tgz -a \"datarobot.com/images\"")
 		assert.NoError(t, err)
 		expectedOutput := `[Dry-Run] Pulling image: docker.io/alpine/curl:8.9.1
 [Dry-Run] ReTagging image: docker.io/alpine/curl:8.9.1 > docker.io/alpine/curl:stable
@@ -28,7 +28,7 @@ func TestCommandSave(t *testing.T) {
 	t.Run("test-chart4", func(t *testing.T) {
 
 		filePath := "image-test.tgz"
-		output, err := executeCommand(rootCmd, "save", "../testdata/test-chart4", "--dry-run=false", "--output", filePath)
+		output, err := executeCommand(rootCmd, "save ../testdata/test-chart4 --dry-run=false --output "+filePath)
 		assert.NoError(t, err)
 
 		// Expected output to compare

cmd/sync_test.go

@@ -8,7 +8,7 @@ import (
 
 func TestCommandSync(t *testing.T) {
 	t.Run("test-chart4", func(t *testing.T) {
-		output, err := executeCommand(rootCmd, "sync", "../testdata/test-chart4", "-r", "registry.example.com", "-u", "testuser", "-p", "testpass", "--dry-run")
+		output, err := executeCommand(rootCmd, "sync ../testdata/test-chart4 -r registry.example.com -u testuser -p testpass --dry-run")
 		assert.NoError(t, err)
 		expectedOutput := `[Dry-Run] Pulling image: docker.io/alpine/curl:8.9.1
 [Dry-Run] Pushing image: registry.example.com/alpine/curl:stable
@@ -23,7 +23,7 @@ func TestCommandSync(t *testing.T) {
 }
 func TestCommandSyncLive(t *testing.T) {
 	t.Run("test-chart4 ttl.sh", func(t *testing.T) {
-		output, err := executeCommand(rootCmd, "sync", "../testdata/test-chart4", "-r", "ttl.sh", "--dry-run=false")
+		output, err := executeCommand(rootCmd, "sync ../testdata/test-chart4 -r ttl.sh --dry-run=false")
 		assert.NoError(t, err)
 		// Expected output to compare
 		expectedOutput := `Pulling image: docker.io/alpine/curl:8.9.1

cmd/utils.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	dr_chartutil "github.com/datarobot-oss/helm-datarobot-plugin/pkg/chartutil"
@@ -20,6 +21,15 @@ func isImageAllowed(image string, imageDoc []dr_chartutil.DatarobotImageDeclarat
 	return false
 }
 
+func SliceHas(slice []string, str string) bool {
+	for _, item := range slice {
+		if item == str {
+			return true
+		}
+	}
+	return false
+}
+
 func ExtractImagesFromManifest(manifest string) ([]string, error) {
 	var manifestImages []string
 
@@ -68,5 +78,7 @@ func ExtractImagesFromManifest(manifest string) ([]string, error) {
 	for _, container := range cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers {
 		manifestImages = append(manifestImages, container.Image)
 	}
+
+	sort.Strings(manifestImages)
 	return manifestImages, nil
 }