Add reader/writer for oci-archive multi image support

Add reader/writer with helpers to allow podman save/load multi oci-archive images.
Allow read oci-archive using source_index to point to the index from oci-archive manifest.
Also reimplement ociArchiveImage{Source,Destination} to support this.

Signed-off-by: Qi Wang <qiwan@redhat.com>
Signed-off-by: Urvashi Mohnani <umohnani@redhat.com>

docs/containers-transports.5.md

@@ -57,14 +57,19 @@ An image stored in the docker daemon's internal storage.
 The image must be specified as a _docker-reference_ or in an alternative _algo:digest_ format when being used as an image source.
 The _algo:digest_ refers to the image ID reported by docker-inspect(1).
 
-### **oci:**_path[:reference]_
+### **oci:**_path[:{reference|@source-index}]_
 
 An image compliant with the "Open Container Image Layout Specification" at _path_.
 Using a _reference_ is optional and allows for storing multiple images at the same _path_.
+For reading images, @_source-index_ is a zero-based index in manifest (to access untagged images).
+If neither reference nor @_source_index is specified when reading an image, the path must contain exactly one image.
 
-### **oci-archive:**_path[:reference]_
+### **oci-archive:**_path[:{reference|@source-index}]_
 
 An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.
+Using a _reference_ is optional and allows for storing multiple images at the same _path_.
+For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images).
+If neither reference nor @_source_index is specified when reading an archive, the archive must contain exactly one image.
 
 ### **ostree:**_docker-reference[@/absolute/repo/path]_
 

oci/archive/oci_dest.go

@@ -2,38 +2,57 @@ package archive
 
 import (
 	"context"
+	"fmt"
 	"io"
-	"os"
 
+	"github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
-	"github.com/containers/storage/pkg/archive"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 type ociArchiveImageDestination struct {
-	ref          ociArchiveReference
-	unpackedDest types.ImageDestination
-	tempDirRef   tempDirOCIRef
+	ref                   ociArchiveReference
+	individualWriterOrNil *Writer
+	unpackedDest          types.ImageDestination
 }
 
 // newImageDestination returns an ImageDestination for writing to an existing directory.
 func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
-	tempDirRef, err := createOCIRef(sys, ref.image)
+	var (
+		archive, individualWriterOrNil *Writer
+		err                            error
+	)
+
+	if ref.sourceIndex != -1 {
+		return nil, fmt.Errorf("destination reference must not contain a manifest index @%d: %w", ref.sourceIndex, invalidOciArchiveErr)
+	}
+
+	if ref.archiveWriter != nil {
+		archive = ref.archiveWriter
+		individualWriterOrNil = nil
+	} else {
+		archive, err = NewWriter(ctx, sys, ref.file)
+		if err != nil {
+			return nil, err
+		}
+		individualWriterOrNil = archive
+	}
+	newref, err := layout.NewReference(archive.tempDir, ref.image)
 	if err != nil {
-		return nil, errors.Wrapf(err, "creating oci reference")
+		archive.Close()
+		return nil, err
 	}
-	unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
+	dst, err := newref.NewImageDestination(ctx, sys)
 	if err != nil {
-		if err := tempDirRef.deleteTempDir(); err != nil {
-			return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
-		}
+		archive.Close()
 		return nil, err
 	}
-	return &ociArchiveImageDestination{ref: ref,
-		unpackedDest: unpackedDest,
-		tempDirRef:   tempDirRef}, nil
+
+	return &ociArchiveImageDestination{
+		unpackedDest:          dst,
+		individualWriterOrNil: individualWriterOrNil,
+		ref:                   ref,
+	}, nil
 }
 
 // Reference returns the reference used to set up this destination.
@@ -42,13 +61,12 @@ func (d *ociArchiveImageDestination) Reference() types.ImageReference {
 }
 
 // Close removes resources associated with an initialized ImageDestination, if any
-// Close deletes the temp directory of the oci-archive image
 func (d *ociArchiveImageDestination) Close() error {
-	defer func() {
-		err := d.tempDirRef.deleteTempDir()
-		logrus.Debugf("Error deleting temporary directory: %v", err)
-	}()
-	return d.unpackedDest.Close()
+	defer d.unpackedDest.Close()
+	if d.ref.archiveWriter != nil || d.individualWriterOrNil == nil {
+		return nil
+	}
+	return d.individualWriterOrNil.Close()
 }
 
 func (d *ociArchiveImageDestination) SupportedManifestMIMETypes() []string {
@@ -135,34 +153,7 @@ func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatur
 // after the directory is made, it is tarred up into a file and the directory is deleted
 func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
 	if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil {
-		return errors.Wrapf(err, "storing image %q", d.ref.image)
+		return fmt.Errorf("storing image %q: %w", d.ref.image, err)
 	}
-
-	// path of directory to tar up
-	src := d.tempDirRef.tempDirectory
-	// path to save tarred up file
-	dst := d.ref.resolvedFile
-	return tarDirectory(src, dst)
-}
-
-// tar converts the directory at src and saves it to dst
-func tarDirectory(src, dst string) error {
-	// input is a stream of bytes from the archive of the directory at path
-	input, err := archive.Tar(src, archive.Uncompressed)
-	if err != nil {
-		return errors.Wrapf(err, "retrieving stream of bytes from %q", src)
-	}
-
-	// creates the tar file
-	outFile, err := os.Create(dst)
-	if err != nil {
-		return errors.Wrapf(err, "creating tar file %q", dst)
-	}
-	defer outFile.Close()
-
-	// copies the contents of the directory to the tar file
-	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
-	_, err = io.Copy(outFile, input)
-
-	return err
+	return nil
 }

oci/archive/oci_src.go

@@ -2,40 +2,67 @@ package archive
 
 import (
 	"context"
+	"fmt"
 	"io"
 
+	"github.com/containers/image/v5/oci/layout"
 	ocilayout "github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
 	digest "github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
 type ociArchiveImageSource struct {
-	ref         ociArchiveReference
-	unpackedSrc types.ImageSource
-	tempDirRef  tempDirOCIRef
+	ref                   ociArchiveReference
+	unpackedSrc           types.ImageSource
+	individualReaderOrNil *Reader
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
-// newImageSource untars the file and saves it in a temp directory
 func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
-	tempDirRef, err := createUntarTempDir(sys, ref)
-	if err != nil {
-		return nil, errors.Wrap(err, "creating temp directory")
+	var (
+		archive, individualReaderOrNil *Reader
+		newref                         types.ImageReference
+		err                            error
+	)
+
+	if ref.archiveReader != nil {
+		archive = ref.archiveReader
+		individualReaderOrNil = nil
+	} else {
+		archive, err = NewReader(ctx, sys, ref)
+		if err != nil {
+			return nil, err
+		}
+		individualReaderOrNil = archive
 	}
 
-	unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
-	if err != nil {
-		if err := tempDirRef.deleteTempDir(); err != nil {
-			return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
+	if ref.sourceIndex != -1 {
+		newref, err = layout.NewIndexReference(archive.tempDirectory, ref.sourceIndex)
+		if err != nil {
+			archive.Close()
+			return nil, err
+		}
+	} else {
+		newref, err = layout.NewReference(archive.tempDirectory, ref.image)
+		if err != nil {
+			archive.Close()
+			return nil, err
 		}
+	}
+
+	src, err := newref.NewImageSource(ctx, sys)
+	if err != nil {
+		archive.Close()
 		return nil, err
 	}
-	return &ociArchiveImageSource{ref: ref,
-		unpackedSrc: unpackedSrc,
-		tempDirRef:  tempDirRef}, nil
+
+	return &ociArchiveImageSource{
+		unpackedSrc:           src,
+		ref:                   ref,
+		individualReaderOrNil: individualReaderOrNil,
+	}, nil
 }
 
 // LoadManifestDescriptor loads the manifest
@@ -48,11 +75,11 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor,
 func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
 	ociArchRef, ok := imgRef.(ociArchiveReference)
 	if !ok {
-		return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociArchiveReference")
+		return imgspecv1.Descriptor{}, fmt.Errorf("error typecasting, need type ociArchiveReference")
 	}
 	tempDirRef, err := createUntarTempDir(sys, ociArchRef)
 	if err != nil {
-		return imgspecv1.Descriptor{}, errors.Wrap(err, "creating temp directory")
+		return imgspecv1.Descriptor{}, fmt.Errorf("creating temp directory: %w", err)
 	}
 	defer func() {
 		err := tempDirRef.deleteTempDir()
@@ -61,7 +88,7 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im
 
 	descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
 	if err != nil {
-		return imgspecv1.Descriptor{}, errors.Wrap(err, "loading index")
+		return imgspecv1.Descriptor{}, fmt.Errorf("loading index: %w", err)
 	}
 	return descriptor, nil
 }
@@ -72,13 +99,12 @@ func (s *ociArchiveImageSource) Reference() types.ImageReference {
 }
 
 // Close removes resources associated with an initialized ImageSource, if any.
-// Close deletes the temporary directory at dst
 func (s *ociArchiveImageSource) Close() error {
-	defer func() {
-		err := s.tempDirRef.deleteTempDir()
-		logrus.Debugf("error deleting tmp dir: %v", err)
-	}()
-	return s.unpackedSrc.Close()
+	defer s.unpackedSrc.Close()
+	if s.ref.archiveReader != nil || s.individualReaderOrNil == nil {
+		return nil
+	}
+	return s.individualReaderOrNil.Close()
 }
 
 // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).