chunked: estargz support

in addition to zstd:chunked, add support for the estargz format.

estargz is maintained at github.com/containerd/stargz-snapshotter

Images using estargz can be used on old clients and registries that
have no support for the zstd compression algorithm.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

pkg/chunked/compression.go

@@ -1,14 +1,18 @@
 package chunked
 
 import (
+	archivetar "archive/tar"
 	"bytes"
 	"encoding/binary"
 	"fmt"
 	"io"
+	"strconv"
 
+	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containers/storage/pkg/chunked/compressor"
 	"github.com/containers/storage/pkg/chunked/internal"
 	"github.com/klauspost/compress/zstd"
+	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/vbatts/tar-split/archive/tar"
@@ -50,25 +54,129 @@ func isZstdChunkedFrameMagic(data []byte) bool {
 	return bytes.Equal(internal.ZstdChunkedFrameMagic, data[:8])
 }
 
+func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) {
+	// information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md
+	footerSize := int64(51)
+	if blobSize <= footerSize {
+		return nil, 0, errors.New("blob too small")
+	}
+	chunk := ImageSourceChunk{
+		Offset: uint64(blobSize - footerSize),
+		Length: uint64(footerSize),
+	}
+	parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
+	if err != nil {
+		return nil, 0, err
+	}
+	var reader io.ReadCloser
+	select {
+	case r := <-parts:
+		reader = r
+	case err := <-errs:
+		return nil, 0, err
+	}
+	defer reader.Close()
+	footer := make([]byte, footerSize)
+	if _, err := io.ReadFull(reader, footer); err != nil {
+		return nil, 0, err
+	}
+
+	/* Read the ToC offset:
+	   - 10 bytes  gzip header
+	   - 2  bytes  XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ"))
+	   - 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
+	   - 2  bytes  Extra: LEN = 22 (16 hex digits + len("STARGZ"))
+	   - 22 bytes  Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC)
+	   - 5  bytes  flate header: BFINAL = 1(last block), BTYPE = 0(non-compressed block), LEN = 0
+	   - 8  bytes  gzip footer
+	*/
+	tocOffset, err := strconv.ParseInt(string(footer[16:16+22-6]), 16, 64)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "parse ToC offset")
+	}
+
+	size := int64(blobSize - footerSize - tocOffset)
+	// set a reasonable limit
+	if size > (1<<20)*50 {
+		return nil, 0, errors.New("manifest too big")
+	}
+
+	chunk = ImageSourceChunk{
+		Offset: uint64(tocOffset),
+		Length: uint64(size),
+	}
+	parts, errs, err = blobStream.GetBlobAt([]ImageSourceChunk{chunk})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	var tocReader io.ReadCloser
+	select {
+	case r := <-parts:
+		tocReader = r
+	case err := <-errs:
+		return nil, 0, err
+	}
+	defer tocReader.Close()
+
+	r, err := pgzip.NewReader(tocReader)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer r.Close()
+
+	aTar := archivetar.NewReader(r)
+
+	header, err := aTar.Next()
+	if err != nil {
+		return nil, 0, err
+	}
+	// set a reasonable limit
+	if header.Size > (1<<20)*50 {
+		return nil, 0, errors.New("manifest too big")
+	}
+
+	manifestUncompressed := make([]byte, header.Size)
+	if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil {
+		return nil, 0, err
+	}
+
+	manifestDigester := digest.Canonical.Digester()
+	manifestChecksum := manifestDigester.Hash()
+	if _, err := manifestChecksum.Write(manifestUncompressed); err != nil {
+		return nil, 0, err
+	}
+
+	d, err := digest.Parse(annotations[estargz.TOCJSONDigestAnnotation])
+	if err != nil {
+		return nil, 0, err
+	}
+	if manifestDigester.Digest() != d {
+		return nil, 0, errors.New("invalid manifest checksum")
+	}
+
+	return manifestUncompressed, tocOffset, nil
+}
+
 // readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream.  The blob total size must
 // be specified.
 // This function uses the io.containers.zstd-chunked. annotations when specified.
-func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, error) {
+func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) {
 	footerSize := int64(internal.FooterSizeSupported)
 	if blobSize <= footerSize {
-		return nil, errors.New("blob too small")
+		return nil, 0, errors.New("blob too small")
 	}
 
 	manifestChecksumAnnotation := annotations[internal.ManifestChecksumKey]
 	if manifestChecksumAnnotation == "" {
-		return nil, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey)
+		return nil, 0, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey)
 	}
 
 	var offset, length, lengthUncompressed, manifestType uint64
 
 	if offsetMetadata := annotations[internal.ManifestInfoKey]; offsetMetadata != "" {
 		if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &offset, &length, &lengthUncompressed, &manifestType); err != nil {
-			return nil, err
+			return nil, 0, err
 		}
 	} else {
 		chunk := ImageSourceChunk{
@@ -77,39 +185,39 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
 		}
 		parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
 		if err != nil {
-			return nil, err
+			return nil, 0, err
 		}
 		var reader io.ReadCloser
 		select {
 		case r := <-parts:
 			reader = r
 		case err := <-errs:
-			return nil, err
+			return nil, 0, err
 		}
 		footer := make([]byte, footerSize)
 		if _, err := io.ReadFull(reader, footer); err != nil {
-			return nil, err
+			return nil, 0, err
 		}
 
 		offset = binary.LittleEndian.Uint64(footer[0:8])
 		length = binary.LittleEndian.Uint64(footer[8:16])
 		lengthUncompressed = binary.LittleEndian.Uint64(footer[16:24])
 		manifestType = binary.LittleEndian.Uint64(footer[24:32])
 		if !isZstdChunkedFrameMagic(footer[32:40]) {
-			return nil, errors.New("invalid magic number")
+			return nil, 0, errors.New("invalid magic number")
 		}
 	}
 
 	if manifestType != internal.ManifestTypeCRFS {
-		return nil, errors.New("invalid manifest type")
+		return nil, 0, errors.New("invalid manifest type")
 	}
 
 	// set a reasonable limit
 	if length > (1<<20)*50 {
-		return nil, errors.New("manifest too big")
+		return nil, 0, errors.New("manifest too big")
 	}
 	if lengthUncompressed > (1<<20)*50 {
-		return nil, errors.New("manifest too big")
+		return nil, 0, errors.New("manifest too big")
 	}
 
 	chunk := ImageSourceChunk{
@@ -119,47 +227,47 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
 
 	parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	var reader io.ReadCloser
 	select {
 	case r := <-parts:
 		reader = r
 	case err := <-errs:
-		return nil, err
+		return nil, 0, err
 	}
 
 	manifest := make([]byte, length)
 	if _, err := io.ReadFull(reader, manifest); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	manifestDigester := digest.Canonical.Digester()
 	manifestChecksum := manifestDigester.Hash()
 	if _, err := manifestChecksum.Write(manifest); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	d, err := digest.Parse(manifestChecksumAnnotation)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if manifestDigester.Digest() != d {
-		return nil, errors.New("invalid manifest checksum")
+		return nil, 0, errors.New("invalid manifest checksum")
 	}
 
 	decoder, err := zstd.NewReader(nil)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	defer decoder.Close()
 
 	b := make([]byte, 0, lengthUncompressed)
 	if decoded, err := decoder.DecodeAll(manifest, b); err == nil {
-		return decoded, nil
+		return decoded, 0, nil
 	}
 
-	return manifest, nil
+	return manifest, int64(offset), nil
 }
 
 // ZstdCompressor is a CompressorFunc for the zstd compression algorithm.

pkg/chunked/compressor/compressor.go

@@ -54,7 +54,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 		return offset, nil
 	}
 
-	var metadata []internal.ZstdFileMetadata
+	var metadata []internal.FileMetadata
 	for {
 		hdr, err := tr.Next()
 		if err != nil {
@@ -116,7 +116,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 		for k, v := range hdr.Xattrs {
 			xattrs[k] = base64.StdEncoding.EncodeToString([]byte(v))
 		}
-		m := internal.ZstdFileMetadata{
+		m := internal.FileMetadata{
 			Type:       typ,
 			Name:       hdr.Name,
 			Linkname:   hdr.Linkname,

pkg/chunked/internal/compression.go

@@ -17,12 +17,12 @@ import (
 	"github.com/opencontainers/go-digest"
 )
 
-type ZstdTOC struct {
-	Version int                `json:"version"`
-	Entries []ZstdFileMetadata `json:"entries"`
+type TOC struct {
+	Version int            `json:"version"`
+	Entries []FileMetadata `json:"entries"`
 }
 
-type ZstdFileMetadata struct {
+type FileMetadata struct {
 	Type       string            `json:"type"`
 	Name       string            `json:"name"`
 	Linkname   string            `json:"linkName,omitempty"`
@@ -114,11 +114,11 @@ func appendZstdSkippableFrame(dest io.Writer, data []byte) error {
 	return nil
 }
 
-func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, offset uint64, metadata []ZstdFileMetadata, level int) error {
+func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, offset uint64, metadata []FileMetadata, level int) error {
 	// 8 is the size of the zstd skippable frame header + the frame size
 	manifestOffset := offset + 8
 
-	toc := ZstdTOC{
+	toc := TOC{
 		Version: 1,
 		Entries: metadata,
 	}