Port Decompression bomb security changes from v1 (#414)

* Add zstd support

Port changes from v1 #292

* Port decompression bomb changes from v1

#412

---------

Co-authored-by: Yan Su <tsu@yaroot.net>
Co-authored-by: Kent 'picat' Gruber <kent@hashicorp.com>

cmd/go-getter/go.sum

@@ -437,6 +437,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ=
+github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

decompress.go

@@ -27,6 +27,7 @@ func init() {
 	tbzDecompressor := new(TarBzip2Decompressor)
 	tgzDecompressor := new(TarGzipDecompressor)
 	txzDecompressor := new(TarXzDecompressor)
+	tzstDecompressor := new(TarZstdDecompressor)
 
 	Decompressors = map[string]Decompressor{
 		"bz2":     new(Bzip2Decompressor),
@@ -35,11 +36,14 @@ func init() {
 		"tar.bz2": tbzDecompressor,
 		"tar.gz":  tgzDecompressor,
 		"tar.xz":  txzDecompressor,
+		"tar.zst": tzstDecompressor,
 		"tbz2":    tbzDecompressor,
 		"tgz":     tgzDecompressor,
 		"txz":     txzDecompressor,
+		"tzst":    tzstDecompressor,
 		"zip":     new(ZipDecompressor),
 		"tar":     tarDecompressor,
+		"zst":     new(ZstdDecompressor),
 	}
 }
 

decompress_bzip2.go

@@ -9,7 +9,12 @@ import (
 
 // Bzip2Decompressor is an implementation of Decompressor that can
 // decompress bz2 files.
-type Bzip2Decompressor struct{}
+type Bzip2Decompressor struct {
+	// FileSizeLimit limits the size of a decompressed file.
+	//
+	// The zero value means no limit.
+	FileSizeLimit int64
+}
 
 func (d *Bzip2Decompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// Directory isn't supported at all
@@ -33,5 +38,5 @@ func (d *Bzip2Decompressor) Decompress(dst, src string, dir bool, umask os.FileM
 	bzipR := bzip2.NewReader(f)
 
 	// Copy it out
-	return copyReader(dst, bzipR, 0622, umask)
+	return copyReader(dst, bzipR, 0622, umask, d.FileSizeLimit)
 }

decompress_gzip.go

@@ -9,7 +9,12 @@ import (
 
 // GzipDecompressor is an implementation of Decompressor that can
 // decompress gzip files.
-type GzipDecompressor struct{}
+type GzipDecompressor struct {
+	// FileSizeLimit limits the size of a decompressed file.
+	//
+	// The zero value means no limit.
+	FileSizeLimit int64
+}
 
 func (d *GzipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// Directory isn't supported at all
@@ -37,5 +42,5 @@ func (d *GzipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMo
 	defer gzipR.Close()
 
 	// Copy it out
-	return copyReader(dst, gzipR, 0622, umask)
+	return copyReader(dst, gzipR, 0622, umask, d.FileSizeLimit)
 }

decompress_tar.go

@@ -11,12 +11,25 @@ import (
 
 // untar is a shared helper for untarring an archive. The reader should provide
 // an uncompressed view of the tar archive.
-func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error {
+func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode, fileSizeLimit int64, filesLimit int) error {
 	tarR := tar.NewReader(input)
 	done := false
 	dirHdrs := []*tar.Header{}
 	now := time.Now()
+
+	var (
+		fileSize   int64
+		filesCount int
+	)
+
 	for {
+		if filesLimit > 0 {
+			filesCount++
+			if filesCount > filesLimit {
+				return fmt.Errorf("tar archive contains too many files: %d > %d", filesCount, filesLimit)
+			}
+		}
+
 		hdr, err := tarR.Next()
 		if err == io.EOF {
 			if !done {
@@ -45,7 +58,15 @@ func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error
 			path = filepath.Join(path, hdr.Name)
 		}
 
-		if hdr.FileInfo().IsDir() {
+		fileInfo := hdr.FileInfo()
+
+		fileSize += fileInfo.Size()
+
+		if fileSizeLimit > 0 && fileSize > fileSizeLimit {
+			return fmt.Errorf("tar archive larger than limit: %d", fileSizeLimit)
+		}
+
+		if fileInfo.IsDir() {
 			if !dir {
 				return fmt.Errorf("expected a single file: %s", src)
 			}
@@ -81,8 +102,8 @@ func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error
 		// Mark that we're done so future in single file mode errors
 		done = true
 
-		// Open the file for writing
-		err = copyReader(path, tarR, hdr.FileInfo().Mode(), umask)
+		// Size limit is tracked using the returned file info.
+		err = copyReader(path, tarR, hdr.FileInfo().Mode(), umask, 0)
 		if err != nil {
 			return err
 		}
@@ -127,7 +148,19 @@ func untar(input io.Reader, dst, src string, dir bool, umask os.FileMode) error
 
 // TarDecompressor is an implementation of Decompressor that can
 // unpack tar files.
-type TarDecompressor struct{}
+type TarDecompressor struct {
+	// FileSizeLimit limits the total size of all
+	// decompressed files.
+	//
+	// The zero value means no limit.
+	FileSizeLimit int64
+
+	// FilesLimit limits the number of files that are
+	// allowed to be decompressed.
+	//
+	// The zero value means no limit.
+	FilesLimit int
+}
 
 func (d *TarDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// If we're going into a directory we should make that first
@@ -146,5 +179,5 @@ func (d *TarDecompressor) Decompress(dst, src string, dir bool, umask os.FileMod
 	}
 	defer f.Close()
 
-	return untar(f, dst, src, dir, umask)
+	return untar(f, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit)
 }

decompress_tar_test.go

@@ -1,10 +1,13 @@
 package getter
 
 import (
+	"archive/tar"
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 )
@@ -45,6 +48,86 @@ func TestTar(t *testing.T) {
 	TestDecompressor(t, new(TarDecompressor), cases)
 }
 
+func TestTarLimits(t *testing.T) {
+	b := bytes.NewBuffer(nil)
+
+	tw := tar.NewWriter(b)
+
+	var files = []struct {
+		Name, Body string
+	}{
+		{"readme.txt", "This archive contains some text files."},
+		{"gopher.txt", "Gopher names:\nCharlie\nRonald\nGlenn"},
+		{"todo.txt", "Get animal handling license."},
+	}
+
+	for _, file := range files {
+		hdr := &tar.Header{
+			Name: file.Name,
+			Mode: 0600,
+			Size: int64(len(file.Body)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tw.Write([]byte(file.Body)); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	td, err := ioutil.TempDir("", "getter")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	tarFilePath := filepath.Join(td, "input.tar")
+
+	err = ioutil.WriteFile(tarFilePath, b.Bytes(), 0666)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	t.Run("file size limit", func(t *testing.T) {
+		d := new(TarDecompressor)
+
+		d.FileSizeLimit = 35
+
+		dst := filepath.Join(td, "subdir", "file-size-limit-result")
+
+		err = d.Decompress(dst, tarFilePath, true, 0022)
+
+		if err == nil {
+			t.Fatal("expected file size limit to error")
+		}
+
+		if !strings.Contains(err.Error(), "tar archive larger than limit: 35") {
+			t.Fatalf("unexpected error: %q", err.Error())
+		}
+	})
+
+	t.Run("files limit", func(t *testing.T) {
+		d := new(TarDecompressor)
+
+		d.FilesLimit = 2
+
+		dst := filepath.Join(td, "subdir", "files-limit-result")
+
+		err = d.Decompress(dst, tarFilePath, true, 0022)
+
+		if err == nil {
+			t.Fatal("expected files limit to error")
+		}
+
+		if !strings.Contains(err.Error(), "tar archive contains too many files: 3 > 2") {
+			t.Fatalf("unexpected error: %q", err.Error())
+		}
+	})
+}
+
 // testDecompressPermissions decompresses a directory and checks the permissions of the expanded files
 func testDecompressorPermissions(t *testing.T, d Decompressor, input string, expected map[string]int, umask os.FileMode) {
 	td, err := ioutil.TempDir("", "getter")

decompress_tbz2.go

@@ -8,7 +8,19 @@ import (
 
 // TarBzip2Decompressor is an implementation of Decompressor that can
 // decompress tar.bz2 files.
-type TarBzip2Decompressor struct{}
+type TarBzip2Decompressor struct {
+	// FileSizeLimit limits the total size of all
+	// decompressed files.
+	//
+	// The zero value means no limit.
+	FileSizeLimit int64
+
+	// FilesLimit limits the number of files that are
+	// allowed to be decompressed.
+	//
+	// The zero value means no limit.
+	FilesLimit int
+}
 
 func (d *TarBzip2Decompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// If we're going into a directory we should make that first
@@ -29,5 +41,5 @@ func (d *TarBzip2Decompressor) Decompress(dst, src string, dir bool, umask os.Fi
 
 	// Bzip2 compression is second
 	bzipR := bzip2.NewReader(f)
-	return untar(bzipR, dst, src, dir, umask)
+	return untar(bzipR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit)
 }

decompress_tgz.go

@@ -9,7 +9,19 @@ import (
 
 // TarGzipDecompressor is an implementation of Decompressor that can
 // decompress tar.gzip files.
-type TarGzipDecompressor struct{}
+type TarGzipDecompressor struct {
+	// FileSizeLimit limits the total size of all
+	// decompressed files.
+	//
+	// The zero value means no limit.
+	FileSizeLimit int64
+
+	// FilesLimit limits the number of files that are
+	// allowed to be decompressed.
+	//
+	// The zero value means no limit.
+	FilesLimit int
+}
 
 func (d *TarGzipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// If we're going into a directory we should make that first
@@ -35,5 +47,5 @@ func (d *TarGzipDecompressor) Decompress(dst, src string, dir bool, umask os.Fil
 	}
 	defer gzipR.Close()
 
-	return untar(gzipR, dst, src, dir, umask)
+	return untar(gzipR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit)
 }

decompress_txz.go

@@ -10,7 +10,19 @@ import (
 
 // TarXzDecompressor is an implementation of Decompressor that can
 // decompress tar.xz files.
-type TarXzDecompressor struct{}
+type TarXzDecompressor struct {
+	// FileSizeLimit limits the total size of all
+	// decompressed files.
+	//
+	// The zero value means no limit.
+	FileSizeLimit int64
+
+	// FilesLimit limits the number of files that are
+	// allowed to be decompressed.
+	//
+	// The zero value means no limit.
+	FilesLimit int
+}
 
 func (d *TarXzDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// If we're going into a directory we should make that first
@@ -35,5 +47,5 @@ func (d *TarXzDecompressor) Decompress(dst, src string, dir bool, umask os.FileM
 		return fmt.Errorf("Error opening an xz reader for %s: %s", src, err)
 	}
 
-	return untar(txzR, dst, src, dir, umask)
+	return untar(txzR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit)
 }

decompress_tzst.go

@@ -0,0 +1,52 @@
+package getter
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+// TarZstdDecompressor is an implementation of Decompressor that can
+// decompress tar.zstd files.
+type TarZstdDecompressor struct {
+	// FileSizeLimit limits the total size of all
+	// decompressed files.
+	//
+	// The zero value means no limit.
+	FileSizeLimit int64
+
+	// FilesLimit limits the number of files that are
+	// allowed to be decompressed.
+	//
+	// The zero value means no limit.
+	FilesLimit int
+}
+
+func (d *TarZstdDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
+	// If we're going into a directory we should make that first
+	mkdir := dst
+	if !dir {
+		mkdir = filepath.Dir(dst)
+	}
+	if err := os.MkdirAll(mkdir, mode(0755, umask)); err != nil {
+		return err
+	}
+
+	// File first
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Zstd compression is second
+	zstdR, err := zstd.NewReader(f)
+	if err != nil {
+		return fmt.Errorf("Error opening a zstd reader for %s: %s", src, err)
+	}
+	defer zstdR.Close()
+
+	return untar(zstdR, dst, src, dir, umask, d.FileSizeLimit, d.FilesLimit)
+}