feat: create index from existing manifests (index create) (#1475)

Signed-off-by: Xiaoxuan Wang <xiaoxuanwang@microsoft.com>

cmd/oras/internal/display/handler.go

@@ -174,6 +174,11 @@ func NewManifestPushHandler(printer *output.Printer) metadata.ManifestPushHandle
 	return text.NewManifestPushHandler(printer)
 }
 
+// NewManifestIndexCreateHandler returns an index create handler.
+func NewManifestIndexCreateHandler(printer *output.Printer) metadata.ManifestIndexCreateHandler {
+	return text.NewManifestIndexCreateHandler(printer)
+}
+
 // NewCopyHandler returns copy handlers.
 func NewCopyHandler(printer *output.Printer, fetcher fetcher.Fetcher) (status.CopyHandler, metadata.CopyHandler) {
 	return status.NewTextCopyHandler(printer, fetcher), text.NewCopyHandler(printer)

cmd/oras/internal/display/metadata/interface.go

@@ -78,6 +78,11 @@ type ManifestPushHandler interface {
 	TaggedHandler
 }
 
+// ManifestIndexCreateHandler handles metadata output for index create events.
+type ManifestIndexCreateHandler interface {
+	TaggedHandler
+}
+
 // CopyHandler handles metadata output for cp events.
 type CopyHandler interface {
 	TaggedHandler

cmd/oras/internal/display/metadata/text/manifest_index_create.go

@@ -0,0 +1,39 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package text
+
+import (
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"oras.land/oras/cmd/oras/internal/display/metadata"
+	"oras.land/oras/cmd/oras/internal/output"
+)
+
+// ManifestIndexCreateHandler handles text metadata output for index create events.
+type ManifestIndexCreateHandler struct {
+	printer *output.Printer
+}
+
+// NewManifestIndexCreateHandler returns a new handler for index create events.
+func NewManifestIndexCreateHandler(printer *output.Printer) metadata.ManifestIndexCreateHandler {
+	return &ManifestIndexCreateHandler{
+		printer: printer,
+	}
+}
+
+// OnTagged implements metadata.TaggedHandler.
+func (h *ManifestIndexCreateHandler) OnTagged(_ ocispec.Descriptor, tag string) error {
+	return h.printer.Println("Tagged", tag)
+}

cmd/oras/internal/display/status/utils.go

@@ -48,6 +48,14 @@ const (
 	copyPromptMounted = "Mounted"
 )
 
+// Prompts for index events.
+const (
+	IndexPromptFetching = "Fetching"
+	IndexPromptFetched  = "Fetched "
+	IndexPromptPacked   = "Packed  "
+	IndexPromptPushed   = "Pushed  "
+)
+
 // DeduplicatedFilter filters out deduplicated descriptors.
 func DeduplicatedFilter(committed *sync.Map) func(desc ocispec.Descriptor) bool {
 	return func(desc ocispec.Descriptor) bool {

cmd/oras/root/manifest/cmd.go

@@ -17,6 +17,7 @@ package manifest
 
 import (
 	"github.com/spf13/cobra"
+	"oras.land/oras/cmd/oras/root/manifest/index"
 )
 
 func Cmd() *cobra.Command {
@@ -30,6 +31,7 @@ func Cmd() *cobra.Command {
 		fetchCmd(),
 		fetchConfigCmd(),
 		pushCmd(),
+		index.Cmd(),
 	)
 	return cmd
 }

cmd/oras/root/manifest/index/cmd.go

@@ -0,0 +1,30 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package index
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func Cmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "index [command]",
+		Short: "[Experimental] Index operations",
+	}
+
+	cmd.AddCommand(
+		createCmd(),
+	)
+	return cmd
+}

cmd/oras/root/manifest/index/create.go

@@ -0,0 +1,181 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package index
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/spf13/cobra"
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/content"
+	"oras.land/oras-go/v2/errdef"
+	"oras.land/oras/cmd/oras/internal/argument"
+	"oras.land/oras/cmd/oras/internal/command"
+	"oras.land/oras/cmd/oras/internal/display"
+	"oras.land/oras/cmd/oras/internal/display/status"
+	oerrors "oras.land/oras/cmd/oras/internal/errors"
+	"oras.land/oras/cmd/oras/internal/option"
+	"oras.land/oras/cmd/oras/internal/output"
+	"oras.land/oras/internal/descriptor"
+	"oras.land/oras/internal/listener"
+)
+
+var maxConfigSize int64 = 4 * 1024 * 1024 // 4 MiB
+
+type createOptions struct {
+	option.Common
+	option.Target
+
+	sources   []string
+	extraRefs []string
+}
+
+func createCmd() *cobra.Command {
+	var opts createOptions
+	cmd := &cobra.Command{
+		Use:   "create [flags] <name>[:<tag[,<tag>][...]] [{<tag>|<digest>}...]",
+		Short: "[Experimental] Create and push an index from provided manifests",
+		Long: `[Experimental] Create and push an index from provided manifests. All manifests should be in the same repository
+
+Example - create an index from source manifests tagged 'linux-amd64' and 'linux-arm64', and push without tagging:
+  oras manifest index create localhost:5000/hello linux-amd64 linux-arm64
+
+Example - create an index from source manifests tagged 'linux-amd64' and 'linux-arm64', and push with the tag 'v1':
+  oras manifest index create localhost:5000/hello:v1 linux-amd64 linux-arm64
+
+Example - create an index from source manifests using both tags and digests, and push with tag 'v1':
+  oras manifest index create localhost:5000/hello:v1 linux-amd64 sha256:99e4703fbf30916f549cd6bfa9cdbab614b5392fbe64fdee971359a77073cdf9
+
+Example - create an index and push it with multiple tags:
+  oras manifest index create localhost:5000/hello:tag1,tag2,tag3 linux-amd64 linux-arm64 sha256:99e4703fbf30916f549cd6bfa9cdbab614b5392fbe64fdee971359a77073cdf9
+
+Example - create an index and push to an OCI image layout folder 'layout-dir' and tag with 'v1':
+  oras manifest index create layout-dir:v1 linux-amd64 sha256:99e4703fbf30916f549cd6bfa9cdbab614b5392fbe64fdee971359a77073cdf9
+`,
+		Args: oerrors.CheckArgs(argument.AtLeast(1), "the destination index to create."),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			refs := strings.Split(args[0], ",")
+			opts.RawReference = refs[0]
+			opts.extraRefs = refs[1:]
+			opts.sources = args[1:]
+			return option.Parse(cmd, &opts)
+		},
+		Aliases: []string{"pack"},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return createIndex(cmd, opts)
+		},
+	}
+	option.ApplyFlags(&opts, cmd.Flags())
+	return oerrors.Command(cmd, &opts.Target)
+}
+
+func createIndex(cmd *cobra.Command, opts createOptions) error {
+	ctx, logger := command.GetLogger(cmd, &opts.Common)
+	target, err := opts.NewTarget(opts.Common, logger)
+	if err != nil {
+		return err
+	}
+	manifests, err := fetchSourceManifests(ctx, target, opts)
+	if err != nil {
+		return err
+	}
+	index := ocispec.Index{
+		Versioned: specs.Versioned{
+			SchemaVersion: 2,
+		},
+		MediaType: ocispec.MediaTypeImageIndex,
+		Manifests: manifests,
+	}
+	indexBytes, _ := json.Marshal(index)
+	desc := content.NewDescriptorFromBytes(ocispec.MediaTypeImageIndex, indexBytes)
+	opts.Println(status.IndexPromptPacked, descriptor.ShortDigest(desc), ocispec.MediaTypeImageIndex)
+	return pushIndex(ctx, target, desc, indexBytes, opts.Reference, opts.extraRefs, opts.AnnotatedReference(), opts.Printer)
+}
+
+func fetchSourceManifests(ctx context.Context, target oras.ReadOnlyTarget, opts createOptions) ([]ocispec.Descriptor, error) {
+	resolved := []ocispec.Descriptor{}
+	for _, source := range opts.sources {
+		opts.Println(status.IndexPromptFetching, source)
+		desc, content, err := oras.FetchBytes(ctx, target, source, oras.DefaultFetchBytesOptions)
+		if err != nil {
+			return nil, fmt.Errorf("could not find the manifest %s: %w", source, err)
+		}
+		if !descriptor.IsManifest(desc) {
+			return nil, fmt.Errorf("%s is not a manifest", source)
+		}
+		opts.Println(status.IndexPromptFetched, source)
+		desc = descriptor.Plain(desc)
+		if descriptor.IsImageManifest(desc) {
+			desc.Platform, err = getPlatform(ctx, target, content)
+			if err != nil {
+				return nil, err
+			}
+		}
+		resolved = append(resolved, desc)
+	}
+	return resolved, nil
+}
+
+func getPlatform(ctx context.Context, target oras.ReadOnlyTarget, manifestBytes []byte) (*ocispec.Platform, error) {
+	// extract config descriptor
+	var manifest ocispec.Manifest
+	if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
+		return nil, err
+	}
+	// if config size is larger than 4 MiB, discontinue the fetch
+	if manifest.Config.Size > maxConfigSize {
+		return nil, fmt.Errorf("config size %v exceeds MaxBytes %v: %w", manifest.Config.Size, maxConfigSize, errdef.ErrSizeExceedsLimit)
+	}
+	// fetch config content
+	contentBytes, err := content.FetchAll(ctx, target, manifest.Config)
+	if err != nil {
+		return nil, err
+	}
+	var platform ocispec.Platform
+	if err := json.Unmarshal(contentBytes, &platform); err != nil || (platform.Architecture == "" && platform.OS == "") {
+		// ignore if the manifest does not have platform information
+		return nil, nil
+	}
+	return &platform, nil
+}
+
+func pushIndex(ctx context.Context, target oras.Target, desc ocispec.Descriptor, content []byte, ref string, extraRefs []string, path string, printer *output.Printer) error {
+	// push the index
+	var err error
+	if ref == "" {
+		err = target.Push(ctx, desc, bytes.NewReader(content))
+	} else {
+		_, err = oras.TagBytes(ctx, target, desc.MediaType, content, ref)
+	}
+	if err != nil {
+		return err
+	}
+	printer.Println(status.IndexPromptPushed, path)
+	if len(extraRefs) != 0 {
+		handler := display.NewManifestIndexCreateHandler(printer)
+		tagListener := listener.NewTaggedListener(target, handler.OnTagged)
+		if _, err = oras.TagBytesN(ctx, tagListener, desc.MediaType, content, extraRefs, oras.DefaultTagBytesNOptions); err != nil {
+			return err
+		}
+	}
+	return printer.Println("Digest:", desc.Digest)
+}

internal/descriptor/descriptor.go

@@ -21,6 +21,20 @@ import (
 	"oras.land/oras/internal/docker"
 )
 
+// IsManifest checks if a descriptor describes a manifest.
+// Adapted from `oras-go`: https://github.com/oras-project/oras-go/blob/d6c837e439f4c567f8003eab6e423c22900452a8/internal/descriptor/descriptor.go#L67
+func IsManifest(desc ocispec.Descriptor) bool {
+	switch desc.MediaType {
+	case docker.MediaTypeManifest,
+		docker.MediaTypeManifestList,
+		ocispec.MediaTypeImageManifest,
+		ocispec.MediaTypeImageIndex:
+		return true
+	default:
+		return false
+	}
+}
+
 // IsImageManifest checks whether a manifest is an image manifest.
 func IsImageManifest(desc ocispec.Descriptor) bool {
 	return desc.MediaType == docker.MediaTypeManifest || desc.MediaType == ocispec.MediaTypeImageManifest
@@ -37,6 +51,16 @@ func ShortDigest(desc ocispec.Descriptor) (digestString string) {
 	return digestString
 }
 
+// Plain returns a plain descriptor that contains only MediaType, Digest and Size.
+// Copied from `oras-go`: https://github.com/oras-project/oras-go/blob/d6c837e439f4c567f8003eab6e423c22900452a8/internal/descriptor/descriptor.go#L81
+func Plain(desc ocispec.Descriptor) ocispec.Descriptor {
+	return ocispec.Descriptor{
+		MediaType: desc.MediaType,
+		Digest:    desc.Digest,
+		Size:      desc.Size,
+	}
+}
+
 // GetTitleOrMediaType gets a descriptor name using either title or media type.
 func GetTitleOrMediaType(desc ocispec.Descriptor) (name string, isTitle bool) {
 	name, ok := desc.Annotations[ocispec.AnnotationTitle]

internal/descriptor/descriptor_test.go

@@ -56,6 +56,18 @@ func TestDescriptor_IsImageManifest(t *testing.T) {
 	}
 }
 
+func TestDescriptor_IsManifest(t *testing.T) {
+	got := descriptor.IsManifest(imageDesc)
+	if !got {
+		t.Fatalf("IsManifest() got %v, want %v", got, true)
+	}
+
+	got = descriptor.IsManifest(artifactDesc)
+	if got {
+		t.Fatalf("IsManifest() got %v, want %v", got, false)
+	}
+}
+
 func TestDescriptor_ShortDigest(t *testing.T) {
 	expected := "2e0e0fe1fb3e"
 	got := descriptor.ShortDigest(titledDesc)

test/e2e/README.md

@@ -163,11 +163,18 @@ graph TD;
         A2-- hello.tar -->A8(blob)
         A3-- hello.tar -->A8(blob)
         A4-- hello.tar -->A8(blob)
+        A9>tag: linux-amd64]-..->A2
+        A10>tag: linux-arm64]-..->A3
+        A11>tag: linux-armv7]-..->A4
 
         B0>tag: foobar]-..->B1[oci image]
         B1-- foo1 -->B2(blob1)
         B1-- foo2 -->B2(blob1)
         B1-- bar -->B3(blob2)
+
+        C0>tag: nonjson-config]-..->C1[oci image]
+        C1-->C2(config4)
+        C1-->C3(blob4)
     end
 ```