From 9d2cc941923f2a0124e713940926b9e14a44278d Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sat, 28 Jan 2023 11:29:17 +0800 Subject: [PATCH 01/19] feat: support discover Signed-off-by: Billy Zha --- cmd/oras/discover.go | 71 +++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/cmd/oras/discover.go b/cmd/oras/discover.go index e34af855f..537afcc83 100644 --- a/cmd/oras/discover.go +++ b/cmd/oras/discover.go @@ -24,8 +24,8 @@ import ( "gopkg.in/yaml.v3" "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/registry/remote" - "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" "github.com/need-being/go-tree" @@ -36,10 +36,9 @@ import ( type discoverOptions struct { option.Common - option.Remote option.Platform + option.Target - targetRef string artifactType string outputType string } @@ -73,10 +72,10 @@ Example - Discover referrers with type 'test-artifact' of manifest 'hello:v1' in `, Args: cobra.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { + opts.RawReference = args[0] return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - opts.targetRef = args[0] return runDiscover(opts) }, } @@ -90,24 +89,24 @@ Example - Discover referrers with type 'test-artifact' of manifest 'hello:v1' in func runDiscover(opts discoverOptions) error { ctx, _ := opts.SetLoggerLevel() - repo, err := opts.NewRepository(opts.targetRef, opts.Common) + repo, err := opts.NewReadonlyTarget(ctx, opts.Common) if err != nil { return err } - if repo.Reference.Reference == "" { - return errors.NewErrInvalidReference(repo.Reference) + if err := opts.EnsureReferenceNotEmpty(); err != nil { + return err } // discover artifacts resolveOpts := oras.DefaultResolveOptions resolveOpts.TargetPlatform = opts.Platform.Platform - desc, err := oras.Resolve(ctx, repo, repo.Reference.Reference, resolveOpts) + desc, err := oras.Resolve(ctx, repo, opts.Reference, resolveOpts) if err != nil { return err } if opts.outputType == "tree" { - root := tree.New(repo.Reference.String()) + root := tree.New(opts.Reference) err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, &opts) if err != nil { return err @@ -124,9 +123,9 @@ func runDiscover(opts discoverOptions) error { } if n := len(refs); n > 1 { - fmt.Println("Discovered", n, "artifacts referencing", repo.Reference) + fmt.Println("Discovered", n, "artifacts referencing", opts.Reference) } else { - fmt.Println("Discovered", n, "artifact referencing", repo.Reference) + fmt.Println("Discovered", n, "artifact referencing", opts.Reference) } fmt.Println("Digest:", desc.Digest) if len(refs) > 0 { @@ -136,19 +135,51 @@ func runDiscover(opts discoverOptions) error { return nil } -func fetchReferrers(ctx context.Context, repo *remote.Repository, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { - results := []ocispec.Descriptor{} - err := repo.Referrers(ctx, desc, artifactType, func(referrers []ocispec.Descriptor) error { - results = append(results, referrers...) - return nil - }) - if err != nil { - return nil, err +func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { + var results []ocispec.Descriptor + if repo, ok := target.(*remote.Repository); ok { + err := repo.Referrers(ctx, desc, artifactType, func(referrers []ocispec.Descriptor) error { + results = append(results, referrers...) + return nil + }) + if err != nil { + return nil, err + } + } else { + // fill in artifact type and filter + predecessors, err := target.Predecessors(ctx, desc) + if err != nil { + return nil, err + } + for _, r := range predecessors { + var fetched []byte + if rc, err := target.Fetch(ctx, r); err != nil { + return nil, err + } else { + fetched, err = content.ReadAll(rc, r) + if err != nil { + return nil, err + } + } + switch r.MediaType { + case ocispec.MediaTypeArtifactManifest: + var artifact ocispec.Artifact + json.Unmarshal(fetched, &artifact) + r.ArtifactType = artifact.ArtifactType + case ocispec.MediaTypeImageManifest: + var image ocispec.Manifest + json.Unmarshal(fetched, &image) + r.ArtifactType = image.Config.MediaType + } + if artifactType == "" || artifactType == r.ArtifactType { + results = append(results, r) + } + } } return results, nil } -func fetchAllReferrers(ctx context.Context, repo *remote.Repository, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error { +func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error { results, err := fetchReferrers(ctx, repo, desc, artifactType) if err != nil { return err From e67be053eb2d86c3117e84a5f0f87a3b897d5108 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sat, 28 Jan 2023 12:31:44 +0800 Subject: [PATCH 02/19] add index support in OCI image layout Signed-off-by: Billy Zha --- cmd/oras/discover.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/oras/discover.go b/cmd/oras/discover.go index 537afcc83..f7ca14cfc 100644 --- a/cmd/oras/discover.go +++ b/cmd/oras/discover.go @@ -170,6 +170,8 @@ func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc o var image ocispec.Manifest json.Unmarshal(fetched, &image) r.ArtifactType = image.Config.MediaType + case ocispec.MediaTypeImageIndex: + r.ArtifactType = "index (manifest list)" } if artifactType == "" || artifactType == r.ArtifactType { results = append(results, r) From 14abb31f07be76613148d9033588ce39285ca069 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sat, 28 Jan 2023 15:22:15 +0800 Subject: [PATCH 03/19] align discover experience to remote registry Signed-off-by: Billy Zha --- cmd/oras/discover.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cmd/oras/discover.go b/cmd/oras/discover.go index f7ca14cfc..ac69b7077 100644 --- a/cmd/oras/discover.go +++ b/cmd/oras/discover.go @@ -138,6 +138,7 @@ func runDiscover(opts discoverOptions) error { func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { var results []ocispec.Descriptor if repo, ok := target.(*remote.Repository); ok { + // get referrers directly err := repo.Referrers(ctx, desc, artifactType, func(referrers []ocispec.Descriptor) error { results = append(results, referrers...) return nil @@ -146,35 +147,38 @@ func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc o return nil, err } } else { - // fill in artifact type and filter + // find matched referrers in all predecessors predecessors, err := target.Predecessors(ctx, desc) if err != nil { return nil, err } - for _, r := range predecessors { + for _, node := range predecessors { var fetched []byte - if rc, err := target.Fetch(ctx, r); err != nil { + if rc, err := target.Fetch(ctx, node); err != nil { return nil, err } else { - fetched, err = content.ReadAll(rc, r) + fetched, err = content.ReadAll(rc, node) if err != nil { return nil, err } } - switch r.MediaType { + var setArtifactType = func(node *ocispec.Descriptor, subject *ocispec.Descriptor, artifactType string) { + if subject != nil && content.Equal(*subject, *node) { + node.ArtifactType = artifactType + } + } + switch node.MediaType { case ocispec.MediaTypeArtifactManifest: var artifact ocispec.Artifact json.Unmarshal(fetched, &artifact) - r.ArtifactType = artifact.ArtifactType + setArtifactType(&node, artifact.Subject, artifact.ArtifactType) case ocispec.MediaTypeImageManifest: var image ocispec.Manifest json.Unmarshal(fetched, &image) - r.ArtifactType = image.Config.MediaType - case ocispec.MediaTypeImageIndex: - r.ArtifactType = "index (manifest list)" + setArtifactType(&node, image.Subject, image.Config.ArtifactType) } - if artifactType == "" || artifactType == r.ArtifactType { - results = append(results, r) + if node.ArtifactType != "" && (artifactType == "" || artifactType == node.ArtifactType) { + results = append(results, node) } } } From 6c103529db8e3f3ddbc6501155b8f421563dd075 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sat, 28 Jan 2023 17:20:44 +0800 Subject: [PATCH 04/19] bug fix Signed-off-by: Billy Zha --- cmd/oras/discover.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/oras/discover.go b/cmd/oras/discover.go index ac69b7077..9fa67556a 100644 --- a/cmd/oras/discover.go +++ b/cmd/oras/discover.go @@ -162,8 +162,8 @@ func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc o return nil, err } } - var setArtifactType = func(node *ocispec.Descriptor, subject *ocispec.Descriptor, artifactType string) { - if subject != nil && content.Equal(*subject, *node) { + var matchSubject = func(got *ocispec.Descriptor, want ocispec.Descriptor, node *ocispec.Descriptor, artifactType string) { + if got != nil && content.Equal(*got, want) { node.ArtifactType = artifactType } } @@ -171,11 +171,11 @@ func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc o case ocispec.MediaTypeArtifactManifest: var artifact ocispec.Artifact json.Unmarshal(fetched, &artifact) - setArtifactType(&node, artifact.Subject, artifact.ArtifactType) + matchSubject(artifact.Subject, desc, &node, artifact.ArtifactType) case ocispec.MediaTypeImageManifest: var image ocispec.Manifest json.Unmarshal(fetched, &image) - setArtifactType(&node, image.Subject, image.Config.ArtifactType) + matchSubject(image.Subject, desc, &node, image.Config.ArtifactType) } if node.ArtifactType != "" && (artifactType == "" || artifactType == node.ArtifactType) { results = append(results, node) From 19014c42e5f8cbb44df3bd27fda756c985741642 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sat, 28 Jan 2023 17:25:31 +0800 Subject: [PATCH 05/19] bug fix Signed-off-by: Billy Zha --- cmd/oras/discover.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/discover.go b/cmd/oras/discover.go index 9fa67556a..f3e74d7ea 100644 --- a/cmd/oras/discover.go +++ b/cmd/oras/discover.go @@ -175,7 +175,7 @@ func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc o case ocispec.MediaTypeImageManifest: var image ocispec.Manifest json.Unmarshal(fetched, &image) - matchSubject(image.Subject, desc, &node, image.Config.ArtifactType) + matchSubject(image.Subject, desc, &node, image.Config.MediaType) } if node.ArtifactType != "" && (artifactType == "" || artifactType == node.ArtifactType) { results = append(results, node) From b1d4c029a744f1dd4048cbedeeaa645681743b8e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sat, 28 Jan 2023 17:35:08 +0800 Subject: [PATCH 06/19] add example Signed-off-by: Billy Zha --- cmd/oras/discover.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/oras/discover.go b/cmd/oras/discover.go index f3e74d7ea..aac7e4ffb 100644 --- a/cmd/oras/discover.go +++ b/cmd/oras/discover.go @@ -69,6 +69,10 @@ Example - Discover all the referrers of manifest with annotations, displayed in Example - Discover referrers with type 'test-artifact' of manifest 'hello:v1' in registry 'localhost:5000': oras discover --artifact-type test-artifact localhost:5000/hello:v1 + +Example - Discover referrers of the manifest tagged 'v1' in an OCI layout folder 'layout-dir': + oras discover layout-dir:v1 + oras discover -v -o tree layout-dir:v1 `, Args: cobra.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { From f0fc6b8a0308c97221048c05edbce1d902e0458e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sat, 28 Jan 2023 17:40:34 +0800 Subject: [PATCH 07/19] add error case Signed-off-by: Billy Zha --- cmd/oras/discover.go | 4 ++++ cmd/oras/internal/option/remote.go | 2 +- cmd/oras/internal/option/spec.go | 18 +++++++++--------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/oras/discover.go b/cmd/oras/discover.go index aac7e4ffb..886df49dc 100644 --- a/cmd/oras/discover.go +++ b/cmd/oras/discover.go @@ -18,6 +18,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "os" "strings" @@ -76,6 +77,9 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI layout folder `, Args: cobra.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { + if opts.ReferrersAPI != nil && opts.Type == option.TargetTypeOCILayout { + return errors.New("cannot specify --distribution-spec flag on an image layout target") + } opts.RawReference = args[0] return option.Parse(&opts) }, diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index caa810b7f..e474c21bc 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -46,11 +46,11 @@ type Remote struct { Username string PasswordFromStdin bool Password string + DistributionSpec resolveFlag []string resolveDialContext func(dialer *net.Dialer) func(context.Context, string, string) (net.Conn, error) applyDistributionSpec bool - distributionSpec distributionSpec } // EnableDistributionSpecFlag set distribution specification flag as applicable. diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index b32ca6062..8a7a10a8e 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -51,27 +51,27 @@ func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { fs.StringVar(&opts.specFlag, "image-spec", "", "specify manifest type for building artifact. options: v1.1-image, v1.1-artifact") } -// distributionSpec option struct. -type distributionSpec struct { - // referrersAPI indicates the preference of the implementation of the Referrers API. +// DistributionSpec option struct. +type DistributionSpec struct { + // ReferrersAPI indicates the preference of the implementation of the Referrers API. // Set to true for referrers API, false for referrers tag scheme, and nil for auto fallback. - referrersAPI *bool + ReferrersAPI *bool // specFlag should be provided in form of`--