Merge pull request #1602 from flouthoc/implement-forcecompressionformat

libimage: add support for `ForceCompressionFormat` for image `copier` and `manifest`

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/containerd/containerd v1.7.3
 	github.com/containernetworking/cni v1.1.2
 	github.com/containernetworking/plugins v1.3.0
-	github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac
+	github.com/containers/image/v5 v5.26.1-0.20230807184415-3fb422379cfa
 	github.com/containers/ocicrypt v1.1.7
 	github.com/containers/storage v1.48.1-0.20230728131509-c3da76fa3f63
 	github.com/coreos/go-systemd/v22 v22.5.0
@@ -60,7 +60,7 @@ require (
 	github.com/containerd/cgroups/v3 v3.0.2 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
-	github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744 // indirect
+	github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd // indirect
 	github.com/docker/docker-credential-helpers v0.8.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -122,9 +122,9 @@ require (
 	go.mongodb.org/mongo-driver v1.11.3 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
+	golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect
 	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/tools v0.9.3 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect

go.sum

@@ -48,8 +48,8 @@ github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl3
 github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
 github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q6mVDp5H1HnjM=
 github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0=
-github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac h1:EQQX+EO+F30H9vJS6vfDXx83Z6OL1YNkO5LN36BtPnM=
-github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac/go.mod h1:Zg7m6YHPZRl/wbUDZ6vt+yAyXAjAvALVUelmsIPpMcE=
+github.com/containers/image/v5 v5.26.1-0.20230807184415-3fb422379cfa h1:wDfVQtc6ik2MvsUmu/YRSyBAE5YUxdjcEDtuT1q2KDo=
+github.com/containers/image/v5 v5.26.1-0.20230807184415-3fb422379cfa/go.mod h1:apL4qwq31NV0gsSZQJPxYyTH0yzWavmMCjT8vsQaXSk=
 github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
 github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U=
@@ -60,8 +60,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744 h1:MqMnhqqfDsYF2bjxndKIqvISTIRBb1KCzrIwVzKJHe0=
-github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
+github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd h1:0av0vtcjA8Hqv5gyWj79CLCFVwOOyBNWPjrfUWceMNg=
+github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
 github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -437,8 +437,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
-golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI=
+golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -461,8 +461,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

libimage/copier.go

@@ -49,6 +49,10 @@ type CopyOptions struct {
 	CompressionFormat *compression.Algorithm
 	// CompressionLevel specifies what compression level is used
 	CompressionLevel *int
+	// ForceCompressionFormat ensures that the compression algorithm set in
+	// CompressionFormat is used exclusively, and blobs of other compression
+	// algorithms are not reused.
+	ForceCompressionFormat bool
 
 	// containers-auth.json(5) file to use when authenticating against
 	// container registries.
@@ -294,6 +298,7 @@ func (r *Runtime) newCopier(options *CopyOptions) (*copier, error) {
 		c.imageCopyOptions.ProgressInterval = time.Second
 	}
 
+	c.imageCopyOptions.ForceCompressionFormat = options.ForceCompressionFormat
 	c.imageCopyOptions.ForceManifestMIMEType = options.ManifestMIMEType
 	c.imageCopyOptions.SourceCtx = c.systemContext
 	c.imageCopyOptions.DestinationCtx = c.systemContext

libimage/manifest_list.go

@@ -461,6 +461,7 @@ func (m *ManifestList) Push(ctx context.Context, destination string, options *Ma
 		SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase,
 		RemoveSignatures:                 options.RemoveSignatures,
 		ManifestType:                     options.ManifestMIMEType,
+		ForceCompressionFormat:           options.ForceCompressionFormat,
 	}
 
 	_, d, err := m.list.Push(ctx, dest, pushOptions)

libimage/manifests/manifests.go

@@ -72,6 +72,7 @@ type PushOptions struct {
 	ManifestType                     string                // the format to use when saving the list - possible options are oci, v2s1, and v2s2
 	SourceFilter                     LookupReferenceFunc   // filter the list source
 	AddCompression                   []string              // add existing instances with requested compression algorithms to manifest list
+	ForceCompressionFormat           bool                  // force push with requested compression ignoring the blobs which can be reused.
 }
 
 // Create creates a new list containing information about the specified image,
@@ -259,6 +260,7 @@ func (l *list) Push(ctx context.Context, dest types.ImageReference, options Push
 		SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase,
 		ForceManifestMIMEType:            singleImageManifestType,
 		EnsureCompressionVariantsExist:   compressionVariants,
+		ForceCompressionFormat:           options.ForceCompressionFormat,
 	}
 
 	// Copy whatever we were asked to copy.

libimage/manifests/manifests_test.go

@@ -337,4 +337,8 @@ func TestPush(t *testing.T) {
 	options.AddCompression = []string{"zstd"}
 	_, _, err = list.Push(ctx, destRef, options)
 	assert.NoError(t, err, "list.Push(with replication for zstd specified)")
+
+	options.ForceCompressionFormat = true
+	_, _, err = list.Push(ctx, destRef, options)
+	assert.NoError(t, err, "list.Push(with ForceCompressionFormat: true)")
 }

libimage/push_test.go

@@ -3,9 +3,13 @@ package libimage
 import (
 	"context"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/containers/common/pkg/config"
+	"github.com/containers/image/v5/pkg/compression"
+	"github.com/containers/image/v5/types"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -96,3 +100,97 @@ func TestPushOtherPlatform(t *testing.T) {
 	_, err = runtime.Push(ctx, "docker.io/library/alpine:latest", "docker-archive:"+tmp.Name(), pushOptions)
 	require.NoError(t, err)
 }
+
+func TestPushWithForceCompression(t *testing.T) {
+	runtime, cleanup := testNewRuntime(t)
+	defer cleanup()
+	ctx := context.Background()
+
+	// Prefetch alpine.
+	pullOptions := &PullOptions{}
+	pullOptions.Writer = os.Stdout
+	pullOptions.Architecture = "arm64"
+	pulledImages, err := runtime.Pull(ctx, "docker.io/library/alpine:latest", config.PullPolicyAlways, pullOptions)
+	require.NoError(t, err)
+	require.Len(t, pulledImages, 1)
+
+	data, err := pulledImages[0].Inspect(ctx, nil)
+	require.NoError(t, err)
+	require.Equal(t, "arm64", data.Architecture)
+
+	// Push newly pulled alpine to directory with uncompressed blobs
+	pushOptions := &PushOptions{}
+	pushOptions.SystemContext = &types.SystemContext{}
+	pushOptions.SystemContext.DirForceDecompress = true
+	pushOptions.Writer = os.Stdout
+	dirDest := t.TempDir()
+	_, err = runtime.Push(ctx, "docker.io/library/alpine:latest", "dir:"+dirDest, pushOptions)
+	require.NoError(t, err)
+
+	// Pull uncompressed alpine from `dir:dirDest` as source.
+	pullOptions = &PullOptions{}
+	pullOptions.Writer = os.Stdout
+	pullOptions.Architecture = "arm64"
+	pulledImages, err = runtime.Pull(ctx, "dir:"+dirDest, config.PullPolicyAlways, pullOptions)
+	require.NoError(t, err)
+	require.Len(t, pulledImages, 1)
+
+	// create `oci` image from uncompressed alpine.
+	pushOptions = &PushOptions{}
+	pushOptions.OciAcceptUncompressedLayers = true
+	pushOptions.Writer = os.Stdout
+	ociDest := t.TempDir()
+	_, err = runtime.Push(ctx, "docker.io/library/alpine:latest", "oci:"+ociDest, pushOptions)
+	require.NoError(t, err)
+
+	// blobs from first push
+	entries, err := os.ReadDir(filepath.Join(ociDest, "blobs", "sha256"))
+	require.NoError(t, err)
+	blobsFirstPush := []string{}
+	for _, e := range entries {
+		blobsFirstPush = append(blobsFirstPush, e.Name())
+	}
+
+	// Note: Compression is changed from `uncompressed` to `Gzip` but blobs
+	// should still be same since `ForceCompressionFormat` is `false`.
+	pushOptions = &PushOptions{}
+	pushOptions.Writer = os.Stdout
+	pushOptions.CompressionFormat = &compression.Gzip
+	pushOptions.ForceCompressionFormat = false
+	_, err = runtime.Push(ctx, "docker.io/library/alpine:latest", "oci:"+ociDest, pushOptions)
+	require.NoError(t, err)
+
+	// blobs from second push
+	entries, err = os.ReadDir(filepath.Join(ociDest, "blobs", "sha256"))
+	require.NoError(t, err)
+	blobsSecondPush := []string{}
+	for _, e := range entries {
+		blobsSecondPush = append(blobsSecondPush, e.Name())
+	}
+
+	// All blobs of first push should be equivalent to blobs of
+	// second push since same compression was used
+	assert.Equal(t, blobsSecondPush, blobsFirstPush)
+
+	// Note: Compression is changed from `uncompressed` to `Gzip` but blobs
+	// should still be same since `ForceCompressionFormat` is `false`.
+	pushOptions = &PushOptions{}
+	pushOptions.Writer = os.Stdout
+	pushOptions.CompressionFormat = &compression.Gzip
+	pushOptions.ForceCompressionFormat = true
+	_, err = runtime.Push(ctx, "docker.io/library/alpine:latest", "oci:"+ociDest, pushOptions)
+	require.NoError(t, err)
+
+	// collect blobs from third push
+	entries, err = os.ReadDir(filepath.Join(ociDest, "blobs", "sha256"))
+	require.NoError(t, err)
+	blobsThirdPush := []string{}
+	for _, e := range entries {
+		blobsThirdPush = append(blobsThirdPush, e.Name())
+	}
+
+	// All blobs of third push should not be equivalent to blobs of
+	// first push since new compression was used and `ForceCompressionFormat`
+	// was `true`.
+	assert.NotEqual(t, blobsThirdPush, blobsFirstPush)
+}