refactor(state.Install): add ability to install Python oneFolder arch…

…ives (#1409)

* refactor(state.Install): add ability to install Python onedir archives

added internal/archive to detect and process tgz files and refactored state.Install to support downloading tgz archives

Signed-off-by: Timothy MacDonald <tim.macdonald@lacework.net>

* style: ran make fmt

Forgot to run make fmt in previons commit

Signed-off-by: Timothy MacDonald <tim.macdonald@lacework.net>

---------

Signed-off-by: Timothy MacDonald <tim.macdonald@lacework.net>

cli/cmd/component_dev.go

@@ -374,7 +374,7 @@ func cdkPythonScaffolding(component *lwcomponent.Component) error {
 		return err
 	}
 	_, err = f.WriteString(fmt.Sprintf(
-		"%s/__main__.py --collect-submodules application -F --name %s --distpath .\"\n",
+		"%s/__main__.py --collect-submodules application -D --name %s --distpath .\"\n",
 		component.Name, component.Name,
 	))
 	if err != nil {

go.mod

@@ -56,6 +56,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/iam v1.18.23
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.33.1
 	github.com/aws/smithy-go v1.13.5
+	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/go-git/go-git/v5 v5.5.2
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/consul/sdk v0.13.1

go.sum

@@ -177,6 +177,8 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
+github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA=
 github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
 github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q=

internal/archive/detect.go

@@ -0,0 +1,89 @@
+package archive
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gabriel-vasile/mimetype"
+)
+
+func detectFileType(file string) (mimeType string, err error) {
+
+	fDescriptor, err := os.Open(file)
+	if err != nil {
+		return
+	}
+	defer fDescriptor.Close()
+	// We only have to pass the file header = first 261 bytes
+	head := make([]byte, 261)
+
+	_, err = fDescriptor.Read(head)
+	if err != nil {
+		return
+	}
+
+	mtype := mimetype.Detect(head)
+	mimeType = mtype.String()
+
+	return
+}
+
+func FileIsGZ(file string) (isGZ bool, err error) {
+	mtype, err := detectFileType(file)
+	if err != nil {
+		return
+	}
+	isGZ = mtype == "application/gzip"
+	return
+}
+
+func FileIsTar(file string) (isTar bool, err error) {
+	mtype, err := detectFileType(file)
+	if err != nil {
+		return
+	}
+	isTar = mtype == "application/x-tar"
+	return
+}
+
+func DetectTGZAndUnpack(filePath string, targetDir string) (err error) {
+	// detect if file is a tar gz and extract to targetDir
+	fileName := filepath.Base(filePath)
+	baseFileName := strings.ReplaceAll(fileName, ".tar.gz", "")
+	baseFileName = strings.ReplaceAll(baseFileName, ".tgz", "")
+	unpackDir, err := os.MkdirTemp("", "temp-cdk-unpack-archive-")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(unpackDir)
+
+	tarFile := filepath.Join(unpackDir, baseFileName+".tar")
+	isGZ, err := FileIsGZ(filePath)
+	if err != nil {
+		return
+	}
+
+	if isGZ {
+		if err := Gunzip(filePath, tarFile); err != nil {
+			return err
+		}
+	} else {
+
+		return
+	}
+
+	isTar, err := FileIsTar(tarFile)
+	if err != nil {
+		return
+	}
+
+	if isTar {
+		if err = UnTar(tarFile, targetDir); err != nil {
+			return
+		}
+	} else {
+		return
+	}
+	return
+}

internal/archive/detect_test.go

@@ -0,0 +1,118 @@
+package archive
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDetectTGZAndUnpackWhenFileIsTGZ(t *testing.T) {
+	testingDir, err := os.MkdirTemp("", "test-cdk-internal-archive-")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(testingDir)
+	testFilePath := filepath.Join("test_resources", "test_archive.tar.gz")
+	err = DetectTGZAndUnpack(testFilePath, testingDir)
+	if err != nil {
+		return
+	}
+
+	extractPath := filepath.Join(testingDir, "test_archive")
+	file, err := os.Open(extractPath)
+	if err != nil {
+		return
+	}
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return
+	}
+	assert.True(t, fileInfo.IsDir())
+}
+
+func TestDetectTGZAndUnpackWhenFileIsNotTGZ(t *testing.T) {
+	testingDir, err := os.MkdirTemp("", "test-cdk-internal-archive-")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(testingDir)
+	testFilePath := filepath.Join(testingDir, "test_file")
+	contents := []byte("testing\n")
+	err = os.WriteFile(testFilePath, contents, 0644)
+	if err != nil {
+		return
+	}
+
+	err = DetectTGZAndUnpack(testFilePath, testingDir)
+	if err != nil {
+		return
+	}
+
+	file, err := os.Open(testFilePath)
+	if err != nil {
+		return
+	}
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return
+	}
+	assert.False(t, fileInfo.IsDir())
+}
+
+func TestFileIsGZWhenFileIsGZ(t *testing.T) {
+	testFilePath := filepath.Join("test_resources", "test_archive.tar.gz")
+	isGZ, err := FileIsGZ(testFilePath)
+	if err != nil {
+		return
+	}
+	assert.True(t, isGZ)
+}
+
+func TestFileIsGZWhenFileIsNotGZ(t *testing.T) {
+	testingDir, err := os.MkdirTemp("", "test-cdk-internal-archive-")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(testingDir)
+	testFilePath := filepath.Join(testingDir, "test_file")
+	contents := []byte("testing\n")
+	err = os.WriteFile(testFilePath, contents, 0644)
+	if err != nil {
+		return
+	}
+	isGZ, err := FileIsGZ(testFilePath)
+	if err != nil {
+		return
+	}
+	assert.False(t, isGZ)
+}
+
+func TestFileIsTarWhenFileIsTar(t *testing.T) {
+	testFilePath := filepath.Join("test_resources", "test_archive.tar")
+	isTar, err := FileIsTar(testFilePath)
+	if err != nil {
+		return
+	}
+	assert.True(t, isTar)
+}
+
+func TestFileIsTarWhenFileIsNotTar(t *testing.T) {
+	testingDir, err := os.MkdirTemp("", "test-cdk-internal-archive-")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(testingDir)
+	testFilePath := filepath.Join(testingDir, "test_file")
+	contents := []byte("testing\n")
+	err = os.WriteFile(testFilePath, contents, 0644)
+	if err != nil {
+		return
+	}
+	isTar, err := FileIsTar(testFilePath)
+	if err != nil {
+		return
+	}
+	assert.False(t, isTar)
+}

internal/archive/gz.go

@@ -0,0 +1,34 @@
+package archive
+
+import (
+	"compress/gzip"
+	"io"
+	"os"
+)
+
+// Inflate GZip file.
+//
+// Writes decompressed data to target path.
+func Gunzip(source string, target string) (err error) {
+	reader, err := os.Open(source)
+	if err != nil {
+		return
+	}
+	defer reader.Close()
+
+	archive, err := gzip.NewReader(reader)
+	if err != nil {
+		return
+	}
+	defer archive.Close()
+
+	writer, err := os.Create(target)
+	if err != nil {
+		return
+	}
+	defer writer.Close()
+
+	_, err = io.Copy(writer, archive)
+
+	return
+}

internal/archive/gz_test.go

@@ -0,0 +1,26 @@
+package archive
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGunzip(t *testing.T) {
+	testingDir, err := os.MkdirTemp("", "test-cdk-internal-archive-")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(testingDir)
+	testGZFile := filepath.Join("test_resources", "test_archives.tar.gz")
+	testTargetFile := filepath.Join(testingDir, "test_archives.tar")
+
+	err = Gunzip(testGZFile, testTargetFile)
+	if err != nil {
+		return
+	}
+	_, err = os.Stat(testTargetFile)
+	assert.True(t, !os.IsNotExist(err))
+}

internal/archive/tar.go

@@ -0,0 +1,64 @@
+package archive
+
+import (
+	"archive/tar"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Extract tarball to dir
+func UnTar(tarball string, dir string) (err error) {
+	reader, err := os.Open(tarball)
+	if err != nil {
+		return
+	}
+	defer reader.Close()
+
+	tarReader := tar.NewReader(reader)
+
+	for {
+		hdr, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return err
+		}
+		path := filepath.Join(dir, hdr.Name)
+		mode := hdr.FileInfo().Mode()
+		switch hdr.Typeflag {
+		case tar.TypeReg:
+			file, err := os.Create(path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+
+			_, err = io.Copy(file, tarReader)
+			if err != nil {
+				return err
+			}
+		case tar.TypeDir:
+			err = os.MkdirAll(path, mode)
+			if err != nil {
+				return err
+			}
+		case tar.TypeLink:
+			err = os.Link(filepath.Join(dir, filepath.Clean(hdr.Linkname)), path)
+			if err != nil {
+				return err
+			}
+		case tar.TypeSymlink:
+			err = os.Symlink(filepath.Clean(hdr.Linkname), path)
+			if err != nil {
+				return err
+			}
+		case tar.TypeXGlobalHeader, tar.TypeXHeader:
+			continue
+
+		}
+
+	}
+
+	return
+}

internal/archive/tar_test.go

@@ -0,0 +1,36 @@
+package archive
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnTar(t *testing.T) {
+	testingDir, err := os.MkdirTemp("", "test-cdk-internal-archive-")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(testingDir)
+	testFilePath := filepath.Join("test_resources", "test_archives.tar")
+	err = UnTar(testFilePath, testingDir)
+	if err != nil {
+		return
+	}
+	regFile := filepath.Join(testingDir, "test_archive", "test_file.txt")
+	dirFile := filepath.Join(testingDir, "test_archive", "test_folder")
+	symlinkFile := filepath.Join(testingDir, "test_archive", "test_soft_line")
+
+	f, err := os.Stat(regFile)
+	isRegFile := !os.IsNotExist(err) && !f.IsDir()
+
+	f, err = os.Stat(dirFile)
+	isDirFile := !os.IsNotExist(err) && f.IsDir()
+
+	f, err = os.Lstat(symlinkFile)
+	isSymlinkFile := !os.IsNotExist(err) && (f.Mode()&os.ModeSymlink == os.ModeSymlink)
+
+	assert.True(t, isRegFile && isDirFile && isSymlinkFile)
+}