From 8ef2943937342a9b8148309cc615d13b967a077d Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 22 Dec 2020 20:21:20 +0900 Subject: [PATCH] containerd-native converter The `nativeconverter` package provides containerd-native converter that does not depend on GGCR. Usage: `ctr images convert --oci --estargz SRC DST` Specifying `--oci` is highly recommended, otherwise "containerd.io/snapshot/stargz/toc.digest" will be lost, unless the original image has already OCI mediatype. Signed-off-by: Akihiro Suda --- cmd/ctr-remote/commands/convert.go | 193 ++++++++ cmd/ctr-remote/main.go | 2 +- nativeconverter/estargz/estargz.go | 121 +++++ nativeconverter/nativeconverter.go | 596 +++++++++++++++++++++++ nativeconverter/uncompress/uncompress.go | 109 +++++ 5 files changed, 1020 insertions(+), 1 deletion(-) create mode 100644 cmd/ctr-remote/commands/convert.go create mode 100644 nativeconverter/estargz/estargz.go create mode 100644 nativeconverter/nativeconverter.go create mode 100644 nativeconverter/uncompress/uncompress.go diff --git a/cmd/ctr-remote/commands/convert.go b/cmd/ctr-remote/commands/convert.go new file mode 100644 index 000000000..59a45cc79 --- /dev/null +++ b/cmd/ctr-remote/commands/convert.go @@ -0,0 +1,193 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commands + +import ( + "compress/gzip" + "encoding/json" + "fmt" + "os" + + "github.com/containerd/containerd/cmd/ctr/commands" + "github.com/containerd/containerd/platforms" + "github.com/containerd/stargz-snapshotter/converter/optimizer/recorder" + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/nativeconverter" + estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz" + "github.com/containerd/stargz-snapshotter/nativeconverter/uncompress" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var ConvertCommand = cli.Command{ + Name: "convert", + Usage: "convert an image", + ArgsUsage: "[flags] ...", + Description: `Convert an image format. + +e.g., 'ctr-remote convert --estargz --oci example.com/foo:orig example.com/foo:esgz' + +Use '--platform' to define the output platform. +When '--all-platforms' is given all images in a manifest list must be available. +`, + Flags: []cli.Flag{ + // estargz flags + cli.BoolFlag{ + Name: "estargz", + Usage: "convert legacy tar(.gz) layers to eStargz for lazy pulling. Should be used in conjunction with '--oci'", + }, + cli.StringFlag{ + Name: "estargz-record-in", + Usage: "Read 'ctr-remote optimize --record-out=' record file", + }, + cli.IntFlag{ + Name: "estargz-compression-level", + Usage: "eStargz compression level", + Value: gzip.BestCompression, + }, + cli.IntFlag{ + Name: "estargz-chunk-size", + Usage: "eStargz chunk size", + Value: 0, + }, + // generic flags + cli.BoolFlag{ + Name: "uncompress", + Usage: "convert tar.gz layers to uncompressed tar layers", + }, + cli.BoolFlag{ + Name: "oci", + Usage: "convert Docker media types to OCI media types", + }, + // platform flags + cli.StringSliceFlag{ + Name: "platform", + Usage: "Pull content from a specific platform", + Value: &cli.StringSlice{}, + }, + cli.BoolFlag{ + Name: "all-platforms", + Usage: "exports content from all platforms", + }, + }, + Action: func(context *cli.Context) error { + var ( + convertOpts = []nativeconverter.ConvertOpt{} + ) + srcRef := context.Args().Get(0) + targetRef := context.Args().Get(1) + if srcRef == "" || targetRef == "" { + return errors.New("src and target image need to be specified") + } + + if !context.Bool("all-platforms") { + if pss := context.StringSlice("platform"); len(pss) > 0 { + var all []ocispec.Platform + for _, ps := range pss { + p, err := platforms.Parse(ps) + if err != nil { + return errors.Wrapf(err, "invalid platform %q", ps) + } + all = append(all, p) + } + convertOpts = append(convertOpts, nativeconverter.WithPlatform(platforms.Ordered(all...))) + } else { + convertOpts = append(convertOpts, nativeconverter.WithPlatform(platforms.Default())) + } + } + + if context.Bool("estargz") { + esgzOpts, err := getESGZConvertOpts(context) + if err != nil { + return err + } + convertOpts = append(convertOpts, nativeconverter.WithLayerConvertFunc(estargzconvert.LayerConvertFunc(esgzOpts...))) + if !context.Bool("oci") { + logrus.Warn("option --estargz should be used in conjunction with --oci") + } + if context.Bool("uncompress") { + return errors.New("option --estargz conflicts with --uncompress") + } + } + + if context.Bool("uncompress") { + convertOpts = append(convertOpts, nativeconverter.WithLayerConvertFunc(uncompress.LayerConvertFunc)) + } + + if context.Bool("oci") { + convertOpts = append(convertOpts, nativeconverter.WithDockerToOCI(true)) + } + + client, ctx, cancel, err := commands.NewClient(context) + if err != nil { + return err + } + defer cancel() + + conv, err := nativeconverter.New(client) + if err != nil { + return err + } + newImg, err := conv.Convert(ctx, targetRef, srcRef, convertOpts...) + if err != nil { + return err + } + fmt.Fprintln(context.App.Writer, newImg.Target.Digest.String()) + return nil + }, +} + +func getESGZConvertOpts(context *cli.Context) ([]estargz.Option, error) { + esgzOpts := []estargz.Option{ + estargz.WithCompressionLevel(context.Int("estargz-compression-level")), + estargz.WithChunkSize(context.Int("estargz-chunk-size")), + } + if estargzRecordIn := context.String("estargz-record-in"); estargzRecordIn != "" { + paths, err := readPathsFromRecordFile(estargzRecordIn) + if err != nil { + return nil, err + } + esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths)) + var ignored []string + esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored)) + } + return esgzOpts, nil +} + +func readPathsFromRecordFile(filename string) ([]string, error) { + r, err := os.Open(filename) + if err != nil { + return nil, err + } + defer r.Close() + dec := json.NewDecoder(r) + var paths []string + added := make(map[string]struct{}) + for dec.More() { + var e recorder.Entry + if err := dec.Decode(&e); err != nil { + return nil, err + } + if _, ok := added[e.Path]; !ok { + paths = append(paths, e.Path) + added[e.Path] = struct{}{} + } + } + return paths, nil +} diff --git a/cmd/ctr-remote/main.go b/cmd/ctr-remote/main.go index 71e4a5c7f..87189a225 100644 --- a/cmd/ctr-remote/main.go +++ b/cmd/ctr-remote/main.go @@ -31,7 +31,7 @@ func init() { } func main() { - customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand} + customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand, commands.ConvertCommand} app := app.New() for i := range app.Commands { if app.Commands[i].Name == "images" { diff --git a/nativeconverter/estargz/estargz.go b/nativeconverter/estargz/estargz.go new file mode 100644 index 000000000..71df22d26 --- /dev/null +++ b/nativeconverter/estargz/estargz.go @@ -0,0 +1,121 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package estargz + +import ( + "context" + "fmt" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containerd/stargz-snapshotter/nativeconverter" + "github.com/containerd/stargz-snapshotter/nativeconverter/uncompress" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// LayerConvertFunc converts legacy tar.gz layers into eStargz tar.gz layers. +// Media type is unchanged. +// +// Should be used in conjunction with WithDockerToOCI(). +// +// Otherwise "containerd.io/snapshot/stargz/toc.digest" annotation will be lost, +// because the Docker media type does not support layer annotations. +func LayerConvertFunc(opts ...estargz.Option) nativeconverter.ConvertFunc { + return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if !images.IsLayerType(desc.MediaType) { + // No conversion. No need to return an error here. + return nil, nil + } + uncompressedDesc := &desc + // We need to uncompress the archive first to call estargz.Build. + if !uncompress.IsUncompressedType(desc.MediaType) { + var err error + uncompressedDesc, err = uncompress.LayerConvertFunc(ctx, cs, desc) + if err != nil { + return nil, err + } + if uncompressedDesc == nil { + return nil, errors.Errorf("unexpectedly got the same blob aftr compression (%s, %q)", desc.Digest, desc.MediaType) + } + defer func() { + if err := cs.Delete(ctx, uncompressedDesc.Digest); err != nil { + logrus.WithError(err).WithField("uncompressedDesc", uncompressedDesc).Warn("failed to remove tmp uncompressed layer") + } + }() + logrus.Debugf("estargz: uncompressed %s into %s", desc.Digest, uncompressedDesc.Digest) + } + + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + labels := info.Labels + + uncompressedReaderAt, err := cs.ReaderAt(ctx, *uncompressedDesc) + if err != nil { + return nil, err + } + defer uncompressedReaderAt.Close() + uncompressedSR := io.NewSectionReader(uncompressedReaderAt, 0, uncompressedDesc.Size) + blob, err := estargz.Build(uncompressedSR, opts...) + if err != nil { + return nil, err + } + defer blob.Close() + ref := fmt.Sprintf("convert-estargz-from-%s", desc.Digest) + w, err := cs.Writer(ctx, content.WithRef(ref)) + if err != nil { + return nil, err + } + defer w.Close() + n, err := io.Copy(w, blob) + if err != nil { + return nil, err + } + if err := blob.Close(); err != nil { + return nil, err + } + // update diffID label + labels[nativeconverter.LabelUncompressed] = blob.DiffID().String() + if err = w.Commit(ctx, n, "", content.WithLabels(labels)); err != nil && !errdefs.IsAlreadyExists(err) { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + newDesc := desc + if uncompress.IsUncompressedType(newDesc.MediaType) { + if nativeconverter.IsDockerType(newDesc.MediaType) { + newDesc.MediaType += ".gzip" + } else { + newDesc.MediaType += "+gzip" + } + } + newDesc.Digest = w.Digest() + newDesc.Size = n + if newDesc.Annotations == nil { + newDesc.Annotations = make(map[string]string, 1) + } + newDesc.Annotations[estargz.TOCJSONDigestAnnotation] = blob.TOCDigest().String() + return &newDesc, nil + } +} diff --git a/nativeconverter/nativeconverter.go b/nativeconverter/nativeconverter.go new file mode 100644 index 000000000..2d6328e29 --- /dev/null +++ b/nativeconverter/nativeconverter.go @@ -0,0 +1,596 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package nativeconverter provides containerd-native converter +// that does not depend on github.com/google/go-containerregistry. +// Most codes except nativeconverter/estargz will be moved to the containerd main repo later. +package nativeconverter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +// New creates a converter. +func New(client *containerd.Client) (*Converter, error) { + conv := &Converter{ + client: client, + } + return conv, nil +} + +// Converter provides converter. +type Converter struct { + client *containerd.Client +} + +type convertOpts struct { + layerConvertFunc ConvertFunc + docker2oci bool + indexConvertFunc ConvertFunc + platformMC platforms.MatchComparer +} + +// ConvertOpt is an option for Convert() +type ConvertOpt func(*convertOpts) error + +// ConvertFunc returns a converted content descriptor. +// When the content was not converted, ConvertContentFunc returns nil. +type ConvertFunc func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) + +// WithLayerConvertFunc specifies the function that converts layers. +func WithLayerConvertFunc(fn ConvertFunc) ConvertOpt { + return func(copts *convertOpts) error { + copts.layerConvertFunc = fn + return nil + } +} + +// WithDockerToOCI converts Docker media types into OCI ones. +func WithDockerToOCI(v bool) ConvertOpt { + return func(copts *convertOpts) error { + copts.docker2oci = true + return nil + } +} + +// WithPlatform specifies the platform. +// Defaults to all platforms. +func WithPlatform(p platforms.MatchComparer) ConvertOpt { + return func(copts *convertOpts) error { + copts.platformMC = p + return nil + } +} + +// WithIndexConvertFunc specifies the function that converts manifests and index (manifest lists). +// Defaults to DefaultIndexConvertFunc. +func WithIndexConvertFunc(fn ConvertFunc) ConvertOpt { + return func(copts *convertOpts) error { + copts.indexConvertFunc = fn + return nil + } +} + +// Convert converts an image. +func (conv *Converter) Convert(ctx context.Context, dstRef, srcRef string, opts ...ConvertOpt) (*images.Image, error) { + var copts convertOpts + for _, o := range opts { + if err := o(&copts); err != nil { + return nil, err + } + } + if copts.platformMC == nil { + copts.platformMC = platforms.All + } + if copts.indexConvertFunc == nil { + copts.indexConvertFunc = DefaultIndexConvertFunc(copts.layerConvertFunc, copts.docker2oci, copts.platformMC) + } + + ctx, done, err := conv.client.WithLease(ctx) + if err != nil { + return nil, err + } + defer done(ctx) + + cs := conv.client.ContentStore() + is := conv.client.ImageService() + srcImg, err := is.Get(ctx, srcRef) + if err != nil { + return nil, err + } + + dstDesc, err := copts.indexConvertFunc(ctx, cs, srcImg.Target) + if err != nil { + return nil, err + } + + dstImg := srcImg + dstImg.Name = dstRef + if dstDesc != nil { + dstImg.Target = *dstDesc + } + var res images.Image + if dstRef != srcRef { + _ = is.Delete(ctx, dstRef) + res, err = is.Create(ctx, dstImg) + } else { + res, err = is.Update(ctx, dstImg) + } + return &res, err +} + +// DefaultIndexConvertFunc is the default convert func. +func DefaultIndexConvertFunc(layerConvertFunc ConvertFunc, docker2oci bool, platformMC platforms.MatchComparer) ConvertFunc { + c := &defaultConverter{ + layerConvertFunc: layerConvertFunc, + docker2oci: docker2oci, + platformMC: platformMC, + diffIDMap: make(map[digest.Digest]digest.Digest), + } + return c.convert +} + +type defaultConverter struct { + layerConvertFunc ConvertFunc + docker2oci bool + platformMC platforms.MatchComparer + diffIDMap map[digest.Digest]digest.Digest // key: old diffID, value: new diffID + diffIDMapMu sync.RWMutex +} + +// convert dispatches desc.MediaType and calls c.convert{Layer,Manifest,Index,Config}. +// +// Also converts media type if c.docker2oci is set. +func (c *defaultConverter) convert(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + newDesc *ocispec.Descriptor + err error + ) + if images.IsLayerType(desc.MediaType) { + newDesc, err = c.convertLayer(ctx, cs, desc) + } else if IsManifestType(desc.MediaType) { + newDesc, err = c.convertManifest(ctx, cs, desc) + } else if IsIndexType(desc.MediaType) { + newDesc, err = c.convertIndex(ctx, cs, desc) + } else if IsConfigType(desc.MediaType) { + newDesc, err = c.convertConfig(ctx, cs, desc) + } + if err != nil { + return nil, err + } + if IsDockerType(desc.MediaType) { + if c.docker2oci { + if newDesc == nil { + newDesc = copyDesc(desc) + } + newDesc.MediaType = ConvertDockerMediaTypeToOCI(newDesc.MediaType) + } else if (newDesc == nil && len(desc.Annotations) != 0) || (newDesc != nil && len(newDesc.Annotations) != 0) { + // Annotations is supported only on OCI manifest. + // We need to remove annotations for Docker media types. + if newDesc == nil { + newDesc = copyDesc(desc) + } + newDesc.Annotations = nil + } + } + logrus.WithField("old", desc).WithField("new", newDesc).Debugf("converted") + return newDesc, nil +} + +func copyDesc(desc ocispec.Descriptor) *ocispec.Descriptor { + descCopy := desc + return &descCopy +} + +// convertLayer converts image image layers if c.layerConvertFunc is set. +// +// c.layerConvertFunc can be nil, e.g., for converting Docker media types to OCI ones. +func (c *defaultConverter) convertLayer(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if c.layerConvertFunc != nil { + return c.layerConvertFunc(ctx, cs, desc) + } + return nil, nil +} + +// convertManifest converts image manifests. +// +// - clears `.mediaType` if the target format is OCI +// +// - records diff ID changes in c.diffIDMap +func (c *defaultConverter) convertManifest(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + manifest DualManifest + modified bool + ) + labels, err := readJSON(ctx, cs, &manifest, desc) + if err != nil { + return nil, err + } + if IsDockerType(manifest.MediaType) && c.docker2oci { + manifest.MediaType = "" + modified = true + } + var mu sync.Mutex + eg, ctx2 := errgroup.WithContext(ctx) + for i, l := range manifest.Layers { + i := i + l := l + oldDiffID, err := GetDiffID(ctx, cs, l) + if err != nil { + return nil, err + } + eg.Go(func() error { + newL, err := c.convert(ctx2, cs, l) + if err != nil { + return err + } + if newL != nil { + mu.Lock() + // update GC labels + ClearGCLabels(labels, l.Digest) + labelKey := fmt.Sprintf("containerd.io/gc.ref.content.l.%d", i) + labels[labelKey] = newL.Digest.String() + manifest.Layers[i] = *newL + modified = true + mu.Unlock() + + // diffID changes if the tar entries were modified. + // diffID stays same if only the compression type was changed. + // When diffID changed, add a map entry so that we can update image config. + newDiffID, err := GetDiffID(ctx, cs, *newL) + if err != nil { + return err + } + if newDiffID != oldDiffID { + c.diffIDMapMu.Lock() + c.diffIDMap[oldDiffID] = newDiffID + c.diffIDMapMu.Unlock() + } + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + + newConfig, err := c.convert(ctx, cs, manifest.Config) + if err != nil { + return nil, err + } + if newConfig != nil { + ClearGCLabels(labels, manifest.Config.Digest) + labels["containerd.io/gc.ref.content.config"] = newConfig.Digest.String() + manifest.Config = *newConfig + modified = true + } + + if modified { + return writeJSON(ctx, cs, &manifest, desc, labels) + } + return nil, nil +} + +// convertIndex converts image index. +// +// - clears `.mediaType` if the target format is OCI +// +// - clears manifest entries that do not match c.platformMC +func (c *defaultConverter) convertIndex(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + index DualIndex + modified bool + ) + labels, err := readJSON(ctx, cs, &index, desc) + if err != nil { + return nil, err + } + if IsDockerType(index.MediaType) && c.docker2oci { + index.MediaType = "" + modified = true + } + + newManifests := make([]ocispec.Descriptor, len(index.Manifests)) + newManifestsToBeRemoved := make(map[int]struct{}) // slice index + var mu sync.Mutex + eg, ctx2 := errgroup.WithContext(ctx) + for i, mani := range index.Manifests { + i := i + mani := mani + labelKey := fmt.Sprintf("containerd.io/gc.ref.content.m.%d", i) + eg.Go(func() error { + if mani.Platform != nil && !c.platformMC.Match(*mani.Platform) { + mu.Lock() + ClearGCLabels(labels, mani.Digest) + newManifestsToBeRemoved[i] = struct{}{} + modified = true + mu.Unlock() + return nil + } + newMani, err := c.convert(ctx2, cs, mani) + if err != nil { + return err + } + mu.Lock() + if newMani != nil { + ClearGCLabels(labels, mani.Digest) + labels[labelKey] = newMani.Digest.String() + // NOTE: for keeping manifest order, we specify `i` index explicitly + newManifests[i] = *newMani + modified = true + } else { + newManifests[i] = mani + } + mu.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + if modified { + var newManifestsClean []ocispec.Descriptor + for i, m := range newManifests { + if _, ok := newManifestsToBeRemoved[i]; !ok { + newManifestsClean = append(newManifestsClean, m) + } + } + index.Manifests = newManifestsClean + return writeJSON(ctx, cs, &index, desc, labels) + } + return nil, nil +} + +// convertConfig converts image config contents. +// +// - updates `.rootfs.diff_ids` using c.diffIDMap . +// +// - clears legacy `.config.Image` and `.container_config.Image` fields if `.rootfs.diff_ids` was updated. +func (c *defaultConverter) convertConfig(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + cfg DualConfig + cfgAsOCI ocispec.Image // read only, used for parsing cfg + modified bool + ) + + labels, err := readJSON(ctx, cs, &cfg, desc) + if err != nil { + return nil, err + } + if _, err := readJSON(ctx, cs, &cfgAsOCI, desc); err != nil { + return nil, err + } + + if rootfs := cfgAsOCI.RootFS; rootfs.Type == "layers" { + rootfsModified := false + c.diffIDMapMu.RLock() + for i, oldDiffID := range rootfs.DiffIDs { + if newDiffID, ok := c.diffIDMap[oldDiffID]; ok && newDiffID != oldDiffID { + rootfs.DiffIDs[i] = newDiffID + rootfsModified = true + } + } + c.diffIDMapMu.RUnlock() + if rootfsModified { + rootfsB, err := json.Marshal(rootfs) + if err != nil { + return nil, err + } + cfg["rootfs"] = (*json.RawMessage)(&rootfsB) + modified = true + } + } + + if modified { + // cfg may have dummy value for legacy `.config.Image` and `.container_config.Image` + // We should clear the ID if we changed the diff IDs. + if _, err := clearDockerV1DummyID(cfg); err != nil { + return nil, err + } + return writeJSON(ctx, cs, &cfg, desc, labels) + } + return nil, nil +} + +// clearDockerV1DummyID clears the dummy values for legacy `.config.Image` and `.container_config.Image`. +// Returns true if the cfg was modified. +func clearDockerV1DummyID(cfg DualConfig) (bool, error) { + var modified bool + f := func(k string) error { + if configX, ok := cfg[k]; ok && configX != nil { + var configField map[string]*json.RawMessage + if err := json.Unmarshal(*configX, &configField); err != nil { + return err + } + delete(configField, "Image") + b, err := json.Marshal(configField) + if err != nil { + return err + } + cfg[k] = (*json.RawMessage)(&b) + modified = true + } + return nil + } + if err := f("config"); err != nil { + return modified, err + } + if err := f("container_config"); err != nil { + return modified, err + } + return modified, nil +} + +type ObjectWithMediaType struct { + // MediaType appears on Docker manifests and manifest lists. + // MediaType does not apper on OCI manifests and index + MediaType string `json:"mediaType,omitempty"` +} + +// DualManifest covers Docker manifest and OCI manifest +type DualManifest struct { + ocispec.Manifest + ObjectWithMediaType +} + +// DualIndex covers Docker manifest list and OCI index +type DualIndex struct { + ocispec.Index + ObjectWithMediaType +} + +// DualConfig covers Docker config (v1.0, v1.1, v1.2) and OCI config. +// Unmarshalled as map[string]*json.RawMessage to retain unknown fields on remarshalling. +type DualConfig map[string]*json.RawMessage + +func readJSON(ctx context.Context, cs content.Store, x interface{}, desc ocispec.Descriptor) (map[string]string, error) { + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + labels := info.Labels + b, err := content.ReadBlob(ctx, cs, desc) + if err != nil { + return nil, err + } + if err := json.Unmarshal(b, x); err != nil { + return nil, err + } + return labels, nil +} + +func writeJSON(ctx context.Context, cs content.Store, x interface{}, oldDesc ocispec.Descriptor, labels map[string]string) (*ocispec.Descriptor, error) { + b, err := json.Marshal(x) + if err != nil { + return nil, err + } + dgst := digest.SHA256.FromBytes(b) + ref := fmt.Sprintf("converter-write-json-%s", dgst.String()) + w, err := content.OpenWriter(ctx, cs, content.WithRef(ref)) + if err != nil { + return nil, err + } + if err := content.Copy(ctx, w, bytes.NewReader(b), int64(len(b)), dgst, content.WithLabels(labels)); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + newDesc := oldDesc + newDesc.Size = int64(len(b)) + newDesc.Digest = dgst + return &newDesc, nil +} + +func IsDockerType(mt string) bool { + return strings.HasPrefix(mt, "application/vnd.docker.") +} + +func IsManifestType(mt string) bool { + switch mt { + case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + // no support for schema1 manifest + return true + default: + return false + } +} + +func IsIndexType(mt string) bool { + switch mt { + case ocispec.MediaTypeImageIndex, images.MediaTypeDockerSchema2ManifestList: + return true + default: + return false + } +} + +func IsConfigType(mt string) bool { + switch mt { + case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: + return true + default: + return false + } +} + +func ConvertDockerMediaTypeToOCI(mt string) string { + switch mt { + case images.MediaTypeDockerSchema2ManifestList: + return ocispec.MediaTypeImageIndex + case images.MediaTypeDockerSchema2Manifest: + return ocispec.MediaTypeImageManifest + case images.MediaTypeDockerSchema2LayerGzip: + return ocispec.MediaTypeImageLayerGzip + case images.MediaTypeDockerSchema2LayerForeignGzip: + return ocispec.MediaTypeImageLayerNonDistributableGzip + case images.MediaTypeDockerSchema2Layer: + return ocispec.MediaTypeImageLayer + case images.MediaTypeDockerSchema2LayerForeign: + return ocispec.MediaTypeImageLayerNonDistributable + case images.MediaTypeDockerSchema2Config: + return ocispec.MediaTypeImageConfig + default: + return mt + } +} + +const LabelUncompressed = "containerd.io/uncompressed" + +func GetDiffID(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (digest.Digest, error) { + switch desc.MediaType { + case + images.MediaTypeDockerSchema2Layer, + ocispec.MediaTypeImageLayer, + images.MediaTypeDockerSchema2LayerForeign, + ocispec.MediaTypeImageLayerNonDistributable: + return desc.Digest, nil + } + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return "", err + } + v, ok := info.Labels[LabelUncompressed] + if !ok { + return "", errors.Wrapf(errdefs.ErrNotFound, "content %s does not have label %q", + desc.Digest.String(), LabelUncompressed) + } + return digest.Parse(v) +} + +func ClearGCLabels(labels map[string]string, dgst digest.Digest) { + for k, v := range labels { + if v == dgst.String() && strings.HasPrefix(k, "containerd.io/gc.ref.content") { + delete(labels, k) + } + } +} diff --git a/nativeconverter/uncompress/uncompress.go b/nativeconverter/uncompress/uncompress.go new file mode 100644 index 000000000..dff821af4 --- /dev/null +++ b/nativeconverter/uncompress/uncompress.go @@ -0,0 +1,109 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package uncompress + +import ( + "compress/gzip" + "context" + "fmt" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/stargz-snapshotter/nativeconverter" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// LayerConvertFunc converts tar.gz layers into uncompressed tar layers. +// Media type is changed, e.g., "application/vnd.oci.image.layer.v1.tar+gzip" -> "application/vnd.oci.image.layer.v1.tar" +func LayerConvertFunc(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if !images.IsLayerType(desc.MediaType) || IsUncompressedType(desc.MediaType) { + // No conversion. No need to return an error here. + return nil, nil + } + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + readerAt, err := cs.ReaderAt(ctx, desc) + if err != nil { + return nil, err + } + defer readerAt.Close() + sr := io.NewSectionReader(readerAt, 0, desc.Size) + newR, err := gzip.NewReader(sr) + if err != nil { + return nil, err + } + defer newR.Close() + ref := fmt.Sprintf("convert-uncompress-from-%s", desc.Digest) + w, err := cs.Writer(ctx, content.WithRef(ref)) + if err != nil { + return nil, err + } + defer w.Close() + n, err := io.Copy(w, newR) + if err != nil { + return nil, err + } + if err := newR.Close(); err != nil { + return nil, err + } + // no need to retain "containerd.io/uncompressed" label, but retain other labels ("containerd.io/distribution.source.*") + labels := info.Labels + delete(labels, nativeconverter.LabelUncompressed) + if err = w.Commit(ctx, 0, "", content.WithLabels(labels)); err != nil && !errdefs.IsAlreadyExists(err) { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + newDesc := desc + newDesc.Digest = w.Digest() + newDesc.Size = n + newDesc.MediaType = convertMediaType(newDesc.MediaType) + return &newDesc, nil +} + +func IsUncompressedType(mt string) bool { + switch mt { + case + images.MediaTypeDockerSchema2Layer, + images.MediaTypeDockerSchema2LayerForeign, + ocispec.MediaTypeImageLayer, + ocispec.MediaTypeImageLayerNonDistributable: + return true + default: + return false + } +} + +func convertMediaType(mt string) string { + switch mt { + case images.MediaTypeDockerSchema2LayerGzip: + return images.MediaTypeDockerSchema2Layer + case images.MediaTypeDockerSchema2LayerForeignGzip: + return images.MediaTypeDockerSchema2LayerForeign + case ocispec.MediaTypeImageLayerGzip: + return ocispec.MediaTypeImageLayer + case ocispec.MediaTypeImageLayerNonDistributableGzip: + return ocispec.MediaTypeImageLayerNonDistributable + default: + return mt + } +}