containerd-native converter

cmd/ctr-remote/commands/convert.go

@@ -0,0 +1,193 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package commands
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/cmd/ctr/commands"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/stargz-snapshotter/converter/optimizer/recorder"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/nativeconverter"
+	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	"github.com/containerd/stargz-snapshotter/nativeconverter/uncompress"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var ConvertCommand = cli.Command{
+	Name:      "convert",
+	Usage:     "convert an image",
+	ArgsUsage: "[flags] <source_ref> <target_ref>...",
+	Description: `Convert an image format.
+
+e.g., 'ctr-remote convert --estargz --oci example.com/foo:orig example.com/foo:esgz'
+
+Use '--platform' to define the output platform.
+When '--all-platforms' is given all images in a manifest list must be available.
+`,
+	Flags: []cli.Flag{
+		// estargz flags
+		cli.BoolFlag{
+			Name:  "estargz",
+			Usage: "convert legacy tar(.gz) layers to eStargz for lazy pulling. Should be used in conjunction with '--oci'",
+		},
+		cli.StringFlag{
+			Name:  "estargz-record-in",
+			Usage: "Read 'ctr-remote optimize --record-out=<FILE>' record file",
+		},
+		cli.IntFlag{
+			Name:  "estargz-compression-level",
+			Usage: "eStargz compression level",
+			Value: gzip.BestCompression,
+		},
+		cli.IntFlag{
+			Name:  "estargz-chunk-size",
+			Usage: "eStargz chunk size",
+			Value: 0,
+		},
+		// generic flags
+		cli.BoolFlag{
+			Name:  "uncompress",
+			Usage: "convert tar.gz layers to uncompressed tar layers",
+		},
+		cli.BoolFlag{
+			Name:  "oci",
+			Usage: "convert Docker media types to OCI media types",
+		},
+		// platform flags
+		cli.StringSliceFlag{
+			Name:  "platform",
+			Usage: "Pull content from a specific platform",
+			Value: &cli.StringSlice{},
+		},
+		cli.BoolFlag{
+			Name:  "all-platforms",
+			Usage: "exports content from all platforms",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			convertOpts = []nativeconverter.ConvertOpt{}
+		)
+		srcRef := context.Args().Get(0)
+		targetRef := context.Args().Get(1)
+		if srcRef == "" || targetRef == "" {
+			return errors.New("src and target image need to be specified")
+		}
+
+		if !context.Bool("all-platforms") {
+			if pss := context.StringSlice("platform"); len(pss) > 0 {
+				var all []ocispec.Platform
+				for _, ps := range pss {
+					p, err := platforms.Parse(ps)
+					if err != nil {
+						return errors.Wrapf(err, "invalid platform %q", ps)
+					}
+					all = append(all, p)
+				}
+				convertOpts = append(convertOpts, nativeconverter.WithPlatform(platforms.Ordered(all...)))
+			} else {
+				convertOpts = append(convertOpts, nativeconverter.WithPlatform(platforms.Default()))
+			}
+		}
+
+		if context.Bool("estargz") {
+			esgzOpts, err := getESGZConvertOpts(context)
+			if err != nil {
+				return err
+			}
+			convertOpts = append(convertOpts, nativeconverter.WithLayerConvertFunc(estargzconvert.LayerConvertFunc(esgzOpts...)))
+			if !context.Bool("oci") {
+				logrus.Warn("option --estargz should be used in conjunction with --oci")
+			}
+			if context.Bool("uncompress") {
+				return errors.New("option --estargz conflicts with --uncompress")
+			}
+		}
+
+		if context.Bool("uncompress") {
+			convertOpts = append(convertOpts, nativeconverter.WithLayerConvertFunc(uncompress.LayerConvertFunc))
+		}
+
+		if context.Bool("oci") {
+			convertOpts = append(convertOpts, nativeconverter.WithDockerToOCI(true))
+		}
+
+		client, ctx, cancel, err := commands.NewClient(context)
+		if err != nil {
+			return err
+		}
+		defer cancel()
+
+		conv, err := nativeconverter.New(client)
+		if err != nil {
+			return err
+		}
+		newImg, err := conv.Convert(ctx, targetRef, srcRef, convertOpts...)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintln(context.App.Writer, newImg.Target.Digest.String())
+		return nil
+	},
+}
+
+func getESGZConvertOpts(context *cli.Context) ([]estargz.Option, error) {
+	esgzOpts := []estargz.Option{
+		estargz.WithCompressionLevel(context.Int("estargz-compression-level")),
+		estargz.WithChunkSize(context.Int("estargz-chunk-size")),
+	}
+	if estargzRecordIn := context.String("estargz-record-in"); estargzRecordIn != "" {
+		paths, err := readPathsFromRecordFile(estargzRecordIn)
+		if err != nil {
+			return nil, err
+		}
+		esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths))
+		var ignored []string
+		esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored))
+	}
+	return esgzOpts, nil
+}
+
+func readPathsFromRecordFile(filename string) ([]string, error) {
+	r, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+	dec := json.NewDecoder(r)
+	var paths []string
+	added := make(map[string]struct{})
+	for dec.More() {
+		var e recorder.Entry
+		if err := dec.Decode(&e); err != nil {
+			return nil, err
+		}
+		if _, ok := added[e.Path]; !ok {
+			paths = append(paths, e.Path)
+			added[e.Path] = struct{}{}
+		}
+	}
+	return paths, nil
+}

cmd/ctr-remote/main.go

@@ -31,7 +31,7 @@ func init() {
 }
 
 func main() {
-	customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand}
+	customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand, commands.ConvertCommand}
 	app := app.New()
 	for i := range app.Commands {
 		if app.Commands[i].Name == "images" {

nativeconverter/estargz/estargz.go

@@ -0,0 +1,121 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package estargz
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/nativeconverter"
+	"github.com/containerd/stargz-snapshotter/nativeconverter/uncompress"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// LayerConvertFunc converts legacy tar.gz layers into eStargz tar.gz layers.
+// Media type is unchanged.
+//
+// Should be used in conjunction with WithDockerToOCI().
+//
+// Otherwise "containerd.io/snapshot/stargz/toc.digest" annotation will be lost,
+// because the Docker media type does not support layer annotations.
+func LayerConvertFunc(opts ...estargz.Option) nativeconverter.ConvertFunc {
+	return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
+		if !images.IsLayerType(desc.MediaType) {
+			// No conversion. No need to return an error here.
+			return nil, nil
+		}
+		uncompressedDesc := &desc
+		// We need to uncompress the archive first to call estargz.Build.
+		if !uncompress.IsUncompressedType(desc.MediaType) {
+			var err error
+			uncompressedDesc, err = uncompress.LayerConvertFunc(ctx, cs, desc)
+			if err != nil {
+				return nil, err
+			}
+			if uncompressedDesc == nil {
+				return nil, errors.Errorf("unexpectedly got the same blob aftr compression (%s, %q)", desc.Digest, desc.MediaType)
+			}
+			defer func() {
+				if err := cs.Delete(ctx, uncompressedDesc.Digest); err != nil {
+					logrus.WithError(err).WithField("uncompressedDesc", uncompressedDesc).Warn("failed to remove tmp uncompressed layer")
+				}
+			}()
+			logrus.Debugf("estargz: uncompressed %s into %s", desc.Digest, uncompressedDesc.Digest)
+		}
+
+		info, err := cs.Info(ctx, desc.Digest)
+		if err != nil {
+			return nil, err
+		}
+		labels := info.Labels
+
+		uncompressedReaderAt, err := cs.ReaderAt(ctx, *uncompressedDesc)
+		if err != nil {
+			return nil, err
+		}
+		defer uncompressedReaderAt.Close()
+		uncompressedSR := io.NewSectionReader(uncompressedReaderAt, 0, uncompressedDesc.Size)
+		blob, err := estargz.Build(uncompressedSR, opts...)
+		if err != nil {
+			return nil, err
+		}
+		defer blob.Close()
+		ref := fmt.Sprintf("convert-estargz-from-%s", desc.Digest)
+		w, err := cs.Writer(ctx, content.WithRef(ref))
+		if err != nil {
+			return nil, err
+		}
+		defer w.Close()
+		n, err := io.Copy(w, blob)
+		if err != nil {
+			return nil, err
+		}
+		if err := blob.Close(); err != nil {
+			return nil, err
+		}
+		// update diffID label
+		labels[nativeconverter.LabelUncompressed] = blob.DiffID().String()
+		if err = w.Commit(ctx, n, "", content.WithLabels(labels)); err != nil && !errdefs.IsAlreadyExists(err) {
+			return nil, err
+		}
+		if err := w.Close(); err != nil {
+			return nil, err
+		}
+		newDesc := desc
+		if uncompress.IsUncompressedType(newDesc.MediaType) {
+			if nativeconverter.IsDockerType(newDesc.MediaType) {
+				newDesc.MediaType += ".gzip"
+			} else {
+				newDesc.MediaType += "+gzip"
+			}
+		}
+		newDesc.Digest = w.Digest()
+		newDesc.Size = n
+		if newDesc.Annotations == nil {
+			newDesc.Annotations = make(map[string]string, 1)
+		}
+		newDesc.Annotations[estargz.TOCJSONDigestAnnotation] = blob.TOCDigest().String()
+		return &newDesc, nil
+	}
+}