From 2cb1a3abfe33d115b73ce9ebf71635834ab01387 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Sat, 20 May 2023 07:06:11 -0400 Subject: [PATCH 1/4] WIP: Quick fix with extra stuff Signed-off-by: Will Murphy --- client.go | 42 +++++++++++++++++++++++++++--- pkg/image/oci/registry_provider.go | 27 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/client.go b/client.go index ce0f4d16..33ab4e5c 100644 --- a/client.go +++ b/client.go @@ -103,15 +103,12 @@ func selectImageProvider(imgStr string, source image.Source, cfg config) (image. var provider image.Provider tempDirGenerator := rootTempDirGenerator.NewGenerator() - if err := setPlatform(source, &cfg, runtime.GOARCH); err != nil { - return nil, err - } - switch source { case image.DockerTarballSource: // note: the imgStr is the path on disk to the tar file provider = docker.NewProviderFromTarball(imgStr, tempDirGenerator) case image.DockerDaemonSource: + // TODO: only specify platform if --platform was passed c, err := dockerClient.GetClient() if err != nil { return nil, err @@ -121,6 +118,7 @@ func selectImageProvider(imgStr string, source image.Source, cfg config) (image. return nil, err } case image.PodmanDaemonSource: + // TODO: only specify platform if --platform was passed c, err := podman.GetClient() if err != nil { return nil, err @@ -134,6 +132,16 @@ func selectImageProvider(imgStr string, source image.Source, cfg config) (image. case image.OciTarballSource: provider = oci.NewProviderFromTarball(imgStr, tempDirGenerator) case image.OciRegistrySource: + // TODO: download manifest, and then do: + // specific platform requested: do that + // no specific platform requested: + // if image is multi-arch: + // try to match host, specify host platform on call to lib + // if not multi-arch, pull the single arch regardless. + + if err := setPlatform(source, &cfg, runtime.GOARCH); err != nil { + return nil, err + } provider = oci.NewProviderFromRegistry(imgStr, tempDirGenerator, cfg.Registry, cfg.Platform) case image.SingularitySource: provider = sif.NewProviderFromPath(imgStr, tempDirGenerator) @@ -179,6 +187,32 @@ func GetImage(ctx context.Context, userStr string, options ...Option) (*image.Im return GetImageFromSource(ctx, imgStr, source, options...) } +func GetImageIndex(ctx context.Context, userStr string, options ...Option) ([]byte, error) { + log.Debugf("userStr is %q", userStr) + _, imgStr, err := image.DetectSource(userStr) + if err != nil { + return nil, err + } + + var cfg config + for _, option := range options { + if option == nil { + continue + } + if err := option(&cfg); err != nil { + return nil, fmt.Errorf("unable to parse option: %w", err) + } + } + log.Debugf("image str is %s", imgStr) + tempDirGenerator := rootTempDirGenerator.NewGenerator() + provider := oci.NewProviderFromRegistry(imgStr, tempDirGenerator, cfg.Registry, cfg.Platform) + index, err := provider.ProvideIndex(ctx, cfg.AdditionalMetadata...) + if err != nil { + return nil, err + } + return index.RawManifest() +} + func SetLogger(logger logger.Logger) { log.Log = logger } diff --git a/pkg/image/oci/registry_provider.go b/pkg/image/oci/registry_provider.go index 3ae3a875..faaa8b10 100644 --- a/pkg/image/oci/registry_provider.go +++ b/pkg/image/oci/registry_provider.go @@ -34,6 +34,20 @@ func NewProviderFromRegistry(imgStr string, tmpDirGen *file.TempDirGenerator, re } } +func (p *RegistryImageProvider) ProvideIndex(ctx context.Context, userMetadata ...image.AdditionalMetadata) (containerregistryV1.ImageIndex, error) { + log.Debugf("pulling index directly from registry image=%q", p.imageStr) + ref, err := name.ParseReference(p.imageStr, prepareReferenceOptions(p.registryOptions)...) + if err != nil { + return nil, err + } + + index, err := remote.Index(ref, prepareRemoteOptions(ctx, ref, p.registryOptions, p.platform)...) + if err != nil { + return nil, err + } + return index, nil +} + // Provide an image object that represents the cached docker image tar fetched a registry. func (p *RegistryImageProvider) Provide(ctx context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { log.Debugf("pulling image info directly from registry image=%q", p.imageStr) @@ -48,6 +62,19 @@ func (p *RegistryImageProvider) Provide(ctx context.Context, userMetadata ...ima return nil, fmt.Errorf("unable to parse registry reference=%q: %+v", p.imageStr, err) } + // TODO: change platform logic here. + + // TODO: download index, and then do: + // specific platform requested: do that + // no specific platform requested: + // if image is multi-arch: + // try to match host, specify host platform on call to lib + // if not multi-arch, pull the single arch regardless. + // probably call https://github.com/google/go-containerregistry/blob/e61c5190e39a37cd78c8c6d71f50a709a9d7eab4/pkg/v1/remote/index.go#L47 + // if arch requested, pull that or fail if unavailable. + // if multi-arch index, and host arch available, pull that + // if multi-arch index, and host arch not available, error and say "ambiguous request"; enumerate available architectures + // if single-arch image, pull that image, but log debug descriptor, err := remote.Get(ref, prepareRemoteOptions(ctx, ref, p.registryOptions, p.platform)...) if err != nil { return nil, fmt.Errorf("failed to get image descriptor from registry: %+v", err) From b453b687acadf36917dc9d5d7977765e7fe84695 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 23 May 2023 10:09:06 -0400 Subject: [PATCH 2/4] WIP - test frame for specific behavior Signed-off-by: Will Murphy --- test/integration/oci_registry_source_test.go | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/integration/oci_registry_source_test.go b/test/integration/oci_registry_source_test.go index feab94d9..ea16c387 100644 --- a/test/integration/oci_registry_source_test.go +++ b/test/integration/oci_registry_source_test.go @@ -44,3 +44,54 @@ func TestOciRegistrySourceMetadata(t *testing.T) { assert.Equal(t, "index.docker.io/"+ref, img.Metadata.RepoDigests[0]) assert.Equal(t, []byte(rawManifest), img.Metadata.RawManifest) } + +func TestOciRegistryArchHandling(t *testing.T) { + // possible platforms are at + // https://github.com/docker/cli/blob/1f6a1a438c4ae426e446f17848114e58072af2bb/cli/command/manifest/util.go#L22 + tests := []struct { + name string + wantErr bool + userStr string + options []stereoscope.Option + }{ + { + name: "arch requested, arch available", + userStr: "registry:alpine:3.18.0", + options: []stereoscope.Option{ + stereoscope.WithPlatform("linux/amd64"), + }, + }, + { + name: "arch requested, arch unavailable", + wantErr: true, + options: []stereoscope.Option{ + stereoscope.WithPlatform("linux/mips64"), + }, + }, + { + name: "multi-arch index, no arch requested, host arch available", + userStr: "registry:alpine:3.18.0", + }, + { + name: "multi-arch index, no arch requested, host arch unavailable", + wantErr: true, + // TODO: this is a really hard one + // maybe make some test images? + }, + { + name: "single arch index", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := stereoscope.GetImage(context.TODO(), tt.userStr) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} From ccc2ba87ee65adf074663dfa565bf8a8cfe0018b Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 30 May 2023 09:20:10 -0400 Subject: [PATCH 3/4] more WIP Signed-off-by: Will Murphy --- client.go | 6 ++++ pkg/image/oci/registry_provider.go | 21 +++++++++++ test/integration/oci_registry_source_test.go | 37 ++++++++++++++++---- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/client.go b/client.go index 33ab4e5c..a38e7ca5 100644 --- a/client.go +++ b/client.go @@ -3,6 +3,7 @@ package stereoscope import ( "context" "fmt" + "os" "runtime" "github.com/wagoodman/go-partybus" @@ -210,6 +211,11 @@ func GetImageIndex(ctx context.Context, userStr string, options ...Option) ([]by if err != nil { return nil, err } + m, err := index.IndexManifest() + for _, man := range m.Manifests { + fmt.Fprintf(os.Stderr, "manifest was: %+v", man) + } + return index.RawManifest() } diff --git a/pkg/image/oci/registry_provider.go b/pkg/image/oci/registry_provider.go index faaa8b10..08b8397d 100644 --- a/pkg/image/oci/registry_provider.go +++ b/pkg/image/oci/registry_provider.go @@ -50,6 +50,7 @@ func (p *RegistryImageProvider) ProvideIndex(ctx context.Context, userMetadata . // Provide an image object that represents the cached docker image tar fetched a registry. func (p *RegistryImageProvider) Provide(ctx context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { + // TODO: at this point, do I know whether platform was requested? I don't think so. log.Debugf("pulling image info directly from registry image=%q", p.imageStr) imageTempDir, err := p.tmpDirGen.NewDirectory("oci-registry-image") @@ -111,6 +112,26 @@ func (p *RegistryImageProvider) Provide(ctx context.Context, userMetadata ...ima return image.New(img, p.tmpDirGen, imageTempDir, metadata...), nil } +func (p *RegistryImageProvider) archIsAvailable(arch string, ref name.Reference, options []remote.Option) (bool, error) { + index, err := remote.Index(ref, options...) + if err != nil { + return false, err + } + manifest, err := index.IndexManifest() + if err != nil { + return false, err + } + for _, m := range manifest.Manifests { + if m.Platform == nil { + continue + } + if m.Platform.Architecture == arch { + return true, nil + } + } + return false, nil +} + func prepareReferenceOptions(registryOptions image.RegistryOptions) []name.Option { var options []name.Option if registryOptions.InsecureUseHTTP { diff --git a/test/integration/oci_registry_source_test.go b/test/integration/oci_registry_source_test.go index ea16c387..cad0cf44 100644 --- a/test/integration/oci_registry_source_test.go +++ b/test/integration/oci_registry_source_test.go @@ -3,6 +3,7 @@ package integration import ( "context" "fmt" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -49,10 +50,11 @@ func TestOciRegistryArchHandling(t *testing.T) { // possible platforms are at // https://github.com/docker/cli/blob/1f6a1a438c4ae426e446f17848114e58072af2bb/cli/command/manifest/util.go#L22 tests := []struct { - name string - wantErr bool - userStr string - options []stereoscope.Option + name string + wantErr bool + userStr string + options []stereoscope.Option + wantArch string }{ { name: "arch requested, arch available", @@ -63,35 +65,56 @@ func TestOciRegistryArchHandling(t *testing.T) { }, { name: "arch requested, arch unavailable", + userStr: "registry:alpine:3.18.0", wantErr: true, options: []stereoscope.Option{ stereoscope.WithPlatform("linux/mips64"), }, }, { - name: "multi-arch index, no arch requested, host arch available", - userStr: "registry:alpine:3.18.0", + name: "multi-arch index, no arch requested, host arch available", + userStr: "registry:alpine:3.18.0", + wantArch: runtime.GOARCH, }, { name: "multi-arch index, no arch requested, host arch unavailable", wantErr: true, + userStr: fmt.Sprintf("registry:%s", getMultiArchImageNotContainingHostArch()), // TODO: this is a really hard one // maybe make some test images? }, { name: "single arch index", wantErr: false, + userStr: fmt.Sprintf("registry:%s", getSingleArchImageNotMatchingHostArch()), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := stereoscope.GetImage(context.TODO(), tt.userStr) + img, err := stereoscope.GetImage(context.TODO(), tt.userStr, tt.options...) if tt.wantErr { assert.Error(t, err) } else { assert.NoError(t, err) } + if tt.wantArch != "" { + assert.Equal(t, tt.wantArch, img.Metadata.Architecture) + } + if img != nil { + t.Logf("%s", img.Metadata.Architecture) + } }) } } + +func getSingleArchImageNotMatchingHostArch() string { + if runtime.GOARCH != "amd64" { + return "rancher/busybox:1.31.1" + } + return "" +} + +func getMultiArchImageNotContainingHostArch() string { + return "" +} From 105f30c129ab8aa2214aee0ee076e208b89da48d Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 30 May 2023 15:16:10 -0400 Subject: [PATCH 4/4] WIP - local multi-arch test Signed-off-by: Will Murphy --- test/integration/oci_registry_source_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/integration/oci_registry_source_test.go b/test/integration/oci_registry_source_test.go index cad0cf44..c7428039 100644 --- a/test/integration/oci_registry_source_test.go +++ b/test/integration/oci_registry_source_test.go @@ -109,6 +109,8 @@ func TestOciRegistryArchHandling(t *testing.T) { } func getSingleArchImageNotMatchingHostArch() string { + // TODO: publish test images to anchore/test_images + // and use those if runtime.GOARCH != "amd64" { return "rancher/busybox:1.31.1" } @@ -116,5 +118,10 @@ func getSingleArchImageNotMatchingHostArch() string { } func getMultiArchImageNotContainingHostArch() string { - return "" + // TODO: publish test images to anchore/test_images + // and use those + if runtime.GOARCH == "arm64" { + return "localhost:5001/learn-multi-arch:no-arm" + } + return "localhost:5001/learn-multi-arch:no-amd64" }