fix/refactor: unpack tar files embedded in layer

If a tar file is embedded in the layer, decompress it and move its contents to the manifests dir.

Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com>

cmd/argocd-repo-server/commands/argocd_repo_server.go

@@ -252,7 +252,7 @@ func NewCommand() *cobra.Command {
 	command.Flags().BoolVar(&disableManifestMaxExtractedSize, "disable-helm-manifest-max-extracted-size", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_HELM_MANIFEST_MAX_EXTRACTED_SIZE", false), "Disable maximum size of helm manifest archives when extracted")
 	command.Flags().BoolVar(&includeHiddenDirectories, "include-hidden-directories", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES", false), "Include hidden directories from Git")
 	command.Flags().BoolVar(&cmpUseManifestGeneratePaths, "plugin-use-manifest-generate-paths", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_PLUGIN_USE_MANIFEST_GENERATE_PATHS", false), "Pass the resources described in argocd.argoproj.io/manifest-generate-paths value to the cmpserver to generate the application manifests.")
-	command.Flags().StringVar(&ociLayerMediaTypes, "oci-layer-media-types", env.StringFromEnv("ARGOCD_REPO_SERVER_OCI_LAYER_MEDIA_TYPES", "application/vnd.oci.image.layer.v1.tar+gzip;application/vnd.cncf.helm.chart.content.v1.tar+gzip"), "Semicolon separated list of allowed media types for OCI media types. This only accounts for media types within layers.")
+	command.Flags().StringVar(&ociLayerMediaTypes, "oci-layer-media-types", env.StringFromEnv("ARGOCD_REPO_SERVER_OCI_LAYER_MEDIA_TYPES", "application/vnd.oci.image.layer.v1.tar;application/vnd.oci.image.layer.v1.tar+gzip;application/vnd.cncf.helm.chart.content.v1.tar+gzip"), "Semicolon separated list of allowed media types for OCI media types. This only accounts for media types within layers.")
 	tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command)
 	cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, cacheutil.Options{
 		OnClientCreated: func(client *redis.Client) {

util/io/files/tar.go

@@ -70,7 +70,7 @@ func writeFile(srcPath string, inclusions []string, exclusions []string, writer
 // Callers must make sure dstPath is:
 //   - a full path
 //   - points to an empty directory or
-//   - points to a non existing directory
+//   - points to a non-existing directory
 func Untgz(dstPath string, r io.Reader, maxSize int64, preserveFileMode bool) error {
 	if !filepath.IsAbs(dstPath) {
 		return fmt.Errorf("dstPath points to a relative path: %s", dstPath)
@@ -81,9 +81,29 @@ func Untgz(dstPath string, r io.Reader, maxSize int64, preserveFileMode bool) er
 		return fmt.Errorf("error reading file: %w", err)
 	}
 	defer gzr.Close()
+	return untar(dstPath, io.LimitReader(gzr, maxSize), preserveFileMode)
+}
 
-	lr := io.LimitReader(gzr, maxSize)
-	tr := tar.NewReader(lr)
+// Untar will loop over the tar reader creating the file structure at dstPath.
+// Callers must make sure dstPath is:
+//   - a full path
+//   - points to an empty directory or
+//   - points to a non-existing directory
+func Untar(dstPath string, r io.Reader, maxSize int64, preserveFileMode bool) error {
+	if !filepath.IsAbs(dstPath) {
+		return fmt.Errorf("dstPath points to a relative path: %s", dstPath)
+	}
+
+	return untar(dstPath, io.LimitReader(r, maxSize), preserveFileMode)
+}
+
+// untar will loop over the tar reader creating the file structure at dstPath.
+// Callers must make sure dstPath is:
+//   - a full path
+//   - points to an empty directory or
+//   - points to a non existing directory
+func untar(dstPath string, r io.Reader, preserveFileMode bool) error {
+	tr := tar.NewReader(r)
 
 	for {
 		header, err := tr.Next()

util/oci/client.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"math"
 	"net/http"
 	"os"
@@ -359,7 +360,7 @@ func isHelmOCI(mediaType string) bool {
 }
 
 func isCompressedLayer(mediaType string) bool {
-	return strings.HasSuffix(mediaType, "tar+gzip")
+	return strings.HasSuffix(mediaType, "tar+gzip") || strings.HasSuffix(mediaType, "tar")
 }
 
 func createTarFile(from, to string) error {
@@ -455,29 +456,61 @@ func (s *compressedLayerExtracterStore) Push(ctx context.Context, desc v1.Descri
 		}
 		defer os.RemoveAll(tempDir)
 
-		err = files.Untgz(tempDir, content, s.maxSize, false)
+		if strings.HasSuffix(desc.MediaType, "tar+gzip") {
+			err = files.Untgz(tempDir, content, s.maxSize, false)
+		} else {
+			err = files.Untar(tempDir, content, s.maxSize, false)
+		}
+
 		if err != nil {
-			return err
+			return fmt.Errorf("could not decompress layer: %w", err)
 		}
 
-		err = os.Remove(s.dest)
+		infos, err := os.ReadDir(tempDir)
 		if err != nil {
 			return err
 		}
 
-		// Helm charts are extracted into a single directory - we want the contents within that directory.
 		if isHelmOCI(desc.MediaType) {
-			infos, err := os.ReadDir(tempDir)
-			if err != nil {
-				return err
-			}
-
 			// For a Helm chart we expect a single tarfile in the directory
 			if len(infos) != 1 {
 				return fmt.Errorf("expected 1 file, found %v", len(infos))
 			}
+		}
+
+		if len(infos) == 1 && infos[0].IsDir() {
+			// Here we assume that this is a directory which has been decompressed. We need to move the contents of
+			// the dir into our intended destination.
+			srcDir := filepath.Join(tempDir, infos[0].Name())
+			return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error {
+				if path != srcDir {
+					// Calculate the relative path from srcDir
+					relPath, err := filepath.Rel(srcDir, path)
+					if err != nil {
+						return err
+					}
+
+					dstPath := filepath.Join(s.dest, relPath)
+					// Move the file by renaming it
+					if d.IsDir() {
+						info, err := d.Info()
+						if err != nil {
+							return err
+						}
+
+						return os.MkdirAll(dstPath, info.Mode())
+					}
+
+					return os.Rename(path, dstPath)
+				}
+
+				return nil
+			})
+		}
 
-			return os.Rename(filepath.Join(tempDir, infos[0].Name()), s.dest)
+		err = os.Remove(s.dest)
+		if err != nil {
+			return err
 		}
 
 		// For any other OCI content, we assume that this should be rendered as-is

util/oci/client_test.go

@@ -112,7 +112,7 @@ func Test_nativeOCIClient_Extract(t *testing.T) {
 				manifestMaxExtractedSize:        10,
 				disableManifestMaxExtractedSize: false,
 			},
-			expectedError: fmt.Errorf("error while iterating on tar reader: unexpected EOF"),
+			expectedError: fmt.Errorf("could not decompress layer: error while iterating on tar reader: unexpected EOF"),
 		},
 		{
 			name: "extraction fails due to multiple layers",
@@ -174,18 +174,19 @@ func Test_nativeOCIClient_Extract(t *testing.T) {
 					tempDir, err := files.CreateTempDir(os.TempDir())
 					defer os.RemoveAll(tempDir)
 					require.NoError(t, err)
-					file, err := os.Open(path)
-					require.NoError(t, err)
-					err = files.Untgz(tempDir, file, math.MaxInt64, false)
-					require.NoError(t, err)
-					chartDir, err := os.ReadDir(tempDir)
+					chartDir, err := os.ReadDir(path)
 					require.NoError(t, err)
 					require.Len(t, chartDir, 1)
-					require.Equal(t, "Chart.yaml", chartDir[0].Name())
-					require.False(t, chartDir[0].IsDir())
-					f, err := os.Open(filepath.Join(tempDir, chartDir[0].Name()))
+					require.Equal(t, "chart.tar.gz", chartDir[0].Name())
+					tarBall, err := os.Open(filepath.Join(path, chartDir[0].Name()))
+					err = files.Untgz(tempDir, tarBall, math.MaxInt64, false)
 					require.NoError(t, err)
-					contents, err := io.ReadAll(f)
+					unpacked, err := os.ReadDir(tempDir)
+					require.Len(t, unpacked, 1)
+					require.Equal(t, "Chart.yaml", unpacked[0].Name())
+					chartYaml, err := os.Open(filepath.Join(tempDir, unpacked[0].Name()))
+					require.NoError(t, err)
+					contents, err := io.ReadAll(chartYaml)
 					require.NoError(t, err)
 					require.Equal(t, "some content", string(contents))
 				},