chunked: estargz support

in addition to zstd:chunked, add support for the estargz format.

estargz is maintained at github.com/containerd/stargz-snapshotter

Images using estargz can be used on old clients and registries that
have no support for the zstd compression algorithm.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/BurntSushi/toml v0.4.1
 	github.com/Microsoft/go-winio v0.5.0
 	github.com/Microsoft/hcsshim v0.8.20
+	github.com/containerd/stargz-snapshotter/estargz v0.7.0
 	github.com/docker/go-units v0.4.0
 	github.com/google/go-intervals v0.0.2
 	github.com/hashicorp/go-multierror v1.1.1

go.sum

@@ -164,6 +164,8 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ
 github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
 github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
+github.com/containerd/stargz-snapshotter/estargz v0.7.0 h1:1d/rydzTywc76lnjJb6qbPCiTiCwts49AzKps/Ecblw=
+github.com/containerd/stargz-snapshotter/estargz v0.7.0/go.mod h1:83VWDqHnurTKliEB0YvWMiCfLDwv4Cjj1X9Vk98GJZw=
 github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
@@ -388,6 +390,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
@@ -711,6 +714,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pkg/chunked/compression.go

@@ -1,14 +1,18 @@
 package chunked
 
 import (
+	archivetar "archive/tar"
 	"bytes"
 	"encoding/binary"
 	"fmt"
 	"io"
+	"strconv"
 
+	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containers/storage/pkg/chunked/compressor"
 	"github.com/containers/storage/pkg/chunked/internal"
 	"github.com/klauspost/compress/zstd"
+	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/vbatts/tar-split/archive/tar"
@@ -50,25 +54,129 @@ func isZstdChunkedFrameMagic(data []byte) bool {
 	return bytes.Equal(internal.ZstdChunkedFrameMagic, data[:8])
 }
 
+func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) {
+	// information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md
+	footerSize := int64(51)
+	if blobSize <= footerSize {
+		return nil, 0, errors.New("blob too small")
+	}
+	chunk := ImageSourceChunk{
+		Offset: uint64(blobSize - footerSize),
+		Length: uint64(footerSize),
+	}
+	parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
+	if err != nil {
+		return nil, 0, err
+	}
+	var reader io.ReadCloser
+	select {
+	case r := <-parts:
+		reader = r
+	case err := <-errs:
+		return nil, 0, err
+	}
+	defer reader.Close()
+	footer := make([]byte, footerSize)
+	if _, err := io.ReadFull(reader, footer); err != nil {
+		return nil, 0, err
+	}
+
+	/* Read the ToC offset:
+	   - 10 bytes  gzip header
+	   - 2  bytes  XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ"))
+	   - 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
+	   - 2  bytes  Extra: LEN = 22 (16 hex digits + len("STARGZ"))
+	   - 22 bytes  Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC)
+	   - 5  bytes  flate header: BFINAL = 1(last block), BTYPE = 0(non-compressed block), LEN = 0
+	   - 8  bytes  gzip footer
+	*/
+	tocOffset, err := strconv.ParseInt(string(footer[16:16+22-6]), 16, 64)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "parse ToC offset")
+	}
+
+	size := int64(blobSize - footerSize - tocOffset)
+	// set a reasonable limit
+	if size > (1<<20)*50 {
+		return nil, 0, errors.New("manifest too big")
+	}
+
+	chunk = ImageSourceChunk{
+		Offset: uint64(tocOffset),
+		Length: uint64(size),
+	}
+	parts, errs, err = blobStream.GetBlobAt([]ImageSourceChunk{chunk})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	var tocReader io.ReadCloser
+	select {
+	case r := <-parts:
+		tocReader = r
+	case err := <-errs:
+		return nil, 0, err
+	}
+	defer tocReader.Close()
+
+	r, err := pgzip.NewReader(tocReader)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer r.Close()
+
+	aTar := archivetar.NewReader(r)
+
+	header, err := aTar.Next()
+	if err != nil {
+		return nil, 0, err
+	}
+	// set a reasonable limit
+	if header.Size > (1<<20)*50 {
+		return nil, 0, errors.New("manifest too big")
+	}
+
+	manifestUncompressed := make([]byte, header.Size)
+	if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil {
+		return nil, 0, err
+	}
+
+	manifestDigester := digest.Canonical.Digester()
+	manifestChecksum := manifestDigester.Hash()
+	if _, err := manifestChecksum.Write(manifestUncompressed); err != nil {
+		return nil, 0, err
+	}
+
+	d, err := digest.Parse(annotations[estargz.TOCJSONDigestAnnotation])
+	if err != nil {
+		return nil, 0, err
+	}
+	if manifestDigester.Digest() != d {
+		return nil, 0, errors.New("invalid manifest checksum")
+	}
+
+	return manifestUncompressed, tocOffset, nil
+}
+
 // readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream.  The blob total size must
 // be specified.
 // This function uses the io.containers.zstd-chunked. annotations when specified.
-func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, error) {
+func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) {
 	footerSize := int64(internal.FooterSizeSupported)
 	if blobSize <= footerSize {
-		return nil, errors.New("blob too small")
+		return nil, 0, errors.New("blob too small")
 	}
 
 	manifestChecksumAnnotation := annotations[internal.ManifestChecksumKey]
 	if manifestChecksumAnnotation == "" {
-		return nil, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey)
+		return nil, 0, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey)
 	}
 
 	var offset, length, lengthUncompressed, manifestType uint64
 
 	if offsetMetadata := annotations[internal.ManifestInfoKey]; offsetMetadata != "" {
 		if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &offset, &length, &lengthUncompressed, &manifestType); err != nil {
-			return nil, err
+			return nil, 0, err
 		}
 	} else {
 		chunk := ImageSourceChunk{
@@ -77,39 +185,39 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
 		}
 		parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
 		if err != nil {
-			return nil, err
+			return nil, 0, err
 		}
 		var reader io.ReadCloser
 		select {
 		case r := <-parts:
 			reader = r
 		case err := <-errs:
-			return nil, err
+			return nil, 0, err
 		}
 		footer := make([]byte, footerSize)
 		if _, err := io.ReadFull(reader, footer); err != nil {
-			return nil, err
+			return nil, 0, err
 		}
 
 		offset = binary.LittleEndian.Uint64(footer[0:8])
 		length = binary.LittleEndian.Uint64(footer[8:16])
 		lengthUncompressed = binary.LittleEndian.Uint64(footer[16:24])
 		manifestType = binary.LittleEndian.Uint64(footer[24:32])
 		if !isZstdChunkedFrameMagic(footer[32:40]) {
-			return nil, errors.New("invalid magic number")
+			return nil, 0, errors.New("invalid magic number")
 		}
 	}
 
 	if manifestType != internal.ManifestTypeCRFS {
-		return nil, errors.New("invalid manifest type")
+		return nil, 0, errors.New("invalid manifest type")
 	}
 
 	// set a reasonable limit
 	if length > (1<<20)*50 {
-		return nil, errors.New("manifest too big")
+		return nil, 0, errors.New("manifest too big")
 	}
 	if lengthUncompressed > (1<<20)*50 {
-		return nil, errors.New("manifest too big")
+		return nil, 0, errors.New("manifest too big")
 	}
 
 	chunk := ImageSourceChunk{
@@ -119,47 +227,47 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
 
 	parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	var reader io.ReadCloser
 	select {
 	case r := <-parts:
 		reader = r
 	case err := <-errs:
-		return nil, err
+		return nil, 0, err
 	}
 
 	manifest := make([]byte, length)
 	if _, err := io.ReadFull(reader, manifest); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	manifestDigester := digest.Canonical.Digester()
 	manifestChecksum := manifestDigester.Hash()
 	if _, err := manifestChecksum.Write(manifest); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	d, err := digest.Parse(manifestChecksumAnnotation)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if manifestDigester.Digest() != d {
-		return nil, errors.New("invalid manifest checksum")
+		return nil, 0, errors.New("invalid manifest checksum")
 	}
 
 	decoder, err := zstd.NewReader(nil)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	defer decoder.Close()
 
 	b := make([]byte, 0, lengthUncompressed)
 	if decoded, err := decoder.DecodeAll(manifest, b); err == nil {
-		return decoded, nil
+		return decoded, 0, nil
 	}
 
-	return manifest, nil
+	return manifest, int64(offset), nil
 }
 
 // ZstdCompressor is a CompressorFunc for the zstd compression algorithm.

pkg/chunked/compressor/compressor.go

@@ -50,7 +50,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 		return offset, nil
 	}
 
-	var metadata []internal.ZstdFileMetadata
+	var metadata []internal.FileMetadata
 	for {
 		hdr, err := tr.Next()
 		if err != nil {
@@ -112,7 +112,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 		for k, v := range hdr.Xattrs {
 			xattrs[k] = base64.StdEncoding.EncodeToString([]byte(v))
 		}
-		m := internal.ZstdFileMetadata{
+		m := internal.FileMetadata{
 			Type:       typ,
 			Name:       hdr.Name,
 			Linkname:   hdr.Linkname,

pkg/chunked/internal/compression.go

@@ -17,12 +17,12 @@ import (
 	"github.com/opencontainers/go-digest"
 )
 
-type ZstdTOC struct {
-	Version int                `json:"version"`
-	Entries []ZstdFileMetadata `json:"entries"`
+type TOC struct {
+	Version int            `json:"version"`
+	Entries []FileMetadata `json:"entries"`
 }
 
-type ZstdFileMetadata struct {
+type FileMetadata struct {
 	Type       string            `json:"type"`
 	Name       string            `json:"name"`
 	Linkname   string            `json:"linkName,omitempty"`
@@ -114,11 +114,11 @@ func appendZstdSkippableFrame(dest io.Writer, data []byte) error {
 	return nil
 }
 
-func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, offset uint64, metadata []ZstdFileMetadata, level int) error {
+func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, offset uint64, metadata []FileMetadata, level int) error {
 	// 8 is the size of the zstd skippable frame header + the frame size
 	manifestOffset := offset + 8
 
-	toc := ZstdTOC{
+	toc := TOC{
 		Version: 1,
 		Entries: metadata,
 	}