chunked: add estargz compressor

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/BurntSushi/toml v0.4.1
 	github.com/Microsoft/go-winio v0.5.0
 	github.com/Microsoft/hcsshim v0.8.20
+	github.com/containerd/stargz-snapshotter/estargz v0.7.0
 	github.com/docker/go-units v0.4.0
 	github.com/google/go-intervals v0.0.2
 	github.com/hashicorp/go-multierror v1.1.1

go.sum

@@ -164,6 +164,8 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ
 github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
 github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
+github.com/containerd/stargz-snapshotter/estargz v0.7.0 h1:1d/rydzTywc76lnjJb6qbPCiTiCwts49AzKps/Ecblw=
+github.com/containerd/stargz-snapshotter/estargz v0.7.0/go.mod h1:83VWDqHnurTKliEB0YvWMiCfLDwv4Cjj1X9Vk98GJZw=
 github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
@@ -388,6 +390,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
@@ -710,6 +713,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pkg/chunked/compressor/compressor.go

@@ -6,11 +6,15 @@ package compressor
 
 import (
 	"encoding/base64"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 
+	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containers/storage/pkg/chunked/internal"
 	"github.com/containers/storage/pkg/ioutils"
+	"github.com/klauspost/pgzip"
 	"github.com/opencontainers/go-digest"
 	"github.com/vbatts/tar-split/archive/tar"
 )
@@ -218,3 +222,89 @@ func ZstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.Wri
 
 	return zstdChunkedWriterWithLevel(r, metadata, *level)
 }
+
+type estarCompressor struct {
+	file             *os.File
+	writer           io.Writer
+	metadata         map[string]string
+	compressionLevel *int
+}
+
+func (c estarCompressor) Write(p []byte) (int, error) {
+	return c.file.Write(p)
+}
+
+func (c estarCompressor) Close() error {
+	defer c.file.Close()
+	st, err := c.file.Stat()
+	if err != nil {
+		return err
+	}
+	sr := io.NewSectionReader(c.file, 0, st.Size())
+
+	compressionLevel := pgzip.BestCompression
+	if c.compressionLevel != nil {
+		compressionLevel = *c.compressionLevel
+	}
+
+	blob, err := estargz.Build(sr, estargz.WithCompressionLevel(compressionLevel))
+	if err != nil {
+		return err
+	}
+	defer blob.Close()
+
+	pr, pw := io.Pipe()
+	uncompressedSizeChan := make(chan int64)
+	go func() {
+		defer pr.Close()
+		defer close(uncompressedSizeChan)
+
+		r, err := pgzip.NewReader(pr)
+		if err != nil {
+			pr.CloseWithError(err)
+			return
+		}
+		defer r.Close()
+
+		nBytes, err := io.Copy(c, r)
+		if err != nil {
+			pr.CloseWithError(err)
+			return
+		}
+		uncompressedSizeChan <- nBytes
+	}()
+
+	if _, err = io.Copy(c.writer, io.TeeReader(blob, pw)); err != nil {
+		pw.Close()
+		return err
+	}
+	if err := pw.Close(); err != nil {
+		return err
+	}
+
+	uncompressedSize := <-uncompressedSizeChan
+
+	c.metadata[estargz.TOCJSONDigestAnnotation] = string(blob.TOCDigest())
+	c.metadata[estargz.StoreUncompressedSizeAnnotation] = fmt.Sprintf("%v", uncompressedSize)
+
+	return err
+}
+
+// EstargzCompressor is a CompressorFunc for the estargz compression algorithm.
+func EstargzCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
+	file, err := ioutil.TempFile("", "estargz-layer")
+	if err != nil {
+		return nil, err
+	}
+	// Unlink immediately the file so we won't leak it.
+	if err := os.Remove(file.Name()); err != nil {
+		return nil, err
+	}
+
+	return estarCompressor{
+		compressionLevel: level,
+		file:             file,
+		writer:           r,
+		metadata:         metadata,
+	}, nil
+}

vendor/modules.txt

@@ -35,6 +35,10 @@ github.com/Microsoft/hcsshim/osversion
 github.com/bits-and-blooms/bitset
 # github.com/containerd/cgroups v1.0.1
 github.com/containerd/cgroups/stats/v1
+# github.com/containerd/stargz-snapshotter/estargz v0.7.0
+## explicit
+github.com/containerd/stargz-snapshotter/estargz
+github.com/containerd/stargz-snapshotter/estargz/errorutil
 # github.com/davecgh/go-spew v1.1.1
 github.com/davecgh/go-spew/spew
 # github.com/docker/go-units v0.4.0
@@ -146,6 +150,8 @@ go.opencensus.io/trace/tracestate
 ## explicit
 golang.org/x/net/context
 # golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55
+# golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+golang.org/x/sync/errgroup
 ## explicit
 golang.org/x/sys/internal/unsafeheader
 golang.org/x/sys/unix