From 82c560715e2f7fb708d3a954a786ca44541934b1 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:22:02 +0100 Subject: [PATCH 01/17] feat: replace docker client code with regclient --- .../extensions/repositories/ocireg/blobs.go | 16 +- .../repositories/ocireg/namespace.go | 12 +- .../repositories/ocireg/repository.go | 124 ++++---- .../extensions/repositories/ocireg/utils.go | 8 +- api/tech/docker/lister.go | 5 +- api/tech/docker/pusher.go | 8 +- api/tech/docker/resolver.go | 10 +- api/tech/oras/client.go | 1 + api/tech/regclient/client.go | 273 ++++++++++++++++++ api/tech/regclient/client_test.go | 13 + .../resolve => regclient}/interface.go | 2 +- go.mod | 5 +- go.sum | 4 + 13 files changed, 392 insertions(+), 89 deletions(-) create mode 100644 api/tech/oras/client.go create mode 100644 api/tech/regclient/client.go create mode 100644 api/tech/regclient/client_test.go rename api/tech/{docker/resolve => regclient}/interface.go (99%) diff --git a/api/oci/extensions/repositories/ocireg/blobs.go b/api/oci/extensions/repositories/ocireg/blobs.go index 0ecf1b299..d65f7e7dc 100644 --- a/api/oci/extensions/repositories/ocireg/blobs.go +++ b/api/oci/extensions/repositories/ocireg/blobs.go @@ -7,10 +7,10 @@ import ( "github.com/mandelsoft/goutils/errors" "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" + "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/extensions/attrs/cacheattr" - "ocm.software/ocm/api/tech/docker/resolve" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" ) @@ -23,20 +23,20 @@ type BlobContainer interface { type blobContainer struct { accessio.StaticAllocatable - fetcher resolve.Fetcher - pusher resolve.Pusher + fetcher regclient.Fetcher + pusher regclient.Pusher mime string } type BlobContainers struct { lock sync.Mutex cache accessio.BlobCache - fetcher resolve.Fetcher - pusher resolve.Pusher + fetcher regclient.Fetcher + pusher regclient.Pusher mimes map[string]BlobContainer } -func NewBlobContainers(ctx cpi.Context, fetcher remotes.Fetcher, pusher resolve.Pusher) *BlobContainers { +func NewBlobContainers(ctx cpi.Context, fetcher remotes.Fetcher, pusher regclient.Pusher) *BlobContainers { return &BlobContainers{ cache: cacheattr.Get(ctx), fetcher: fetcher, @@ -73,7 +73,7 @@ func (c *BlobContainers) Release() error { return list.Result() } -func newBlobContainer(mime string, fetcher resolve.Fetcher, pusher resolve.Pusher) *blobContainer { +func newBlobContainer(mime string, fetcher regclient.Fetcher, pusher regclient.Pusher) *blobContainer { return &blobContainer{ mime: mime, fetcher: fetcher, @@ -81,7 +81,7 @@ func newBlobContainer(mime string, fetcher resolve.Fetcher, pusher resolve.Pushe } } -func NewBlobContainer(cache accessio.BlobCache, mime string, fetcher resolve.Fetcher, pusher resolve.Pusher) (BlobContainer, error) { +func NewBlobContainer(cache accessio.BlobCache, mime string, fetcher regclient.Fetcher, pusher regclient.Pusher) (BlobContainer, error) { c := newBlobContainer(mime, fetcher, pusher) if cache == nil { diff --git a/api/oci/extensions/repositories/ocireg/namespace.go b/api/oci/extensions/repositories/ocireg/namespace.go index 9ef823997..43477b011 100644 --- a/api/oci/extensions/repositories/ocireg/namespace.go +++ b/api/oci/extensions/repositories/ocireg/namespace.go @@ -7,12 +7,12 @@ import ( "github.com/containerd/errdefs" "github.com/mandelsoft/goutils/errors" "github.com/opencontainers/go-digest" + "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/cpi/support" "ocm.software/ocm/api/oci/extensions/actions/oci-repository-prepare" - "ocm.software/ocm/api/tech/docker/resolve" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" "ocm.software/ocm/api/utils/logging" @@ -22,10 +22,10 @@ import ( type NamespaceContainer struct { impl support.NamespaceAccessImpl repo *RepositoryImpl - resolver resolve.Resolver - lister resolve.Lister - fetcher resolve.Fetcher - pusher resolve.Pusher + resolver regclient.Resolver + lister regclient.Lister + fetcher regclient.Fetcher + pusher regclient.Pusher blobs *BlobContainers checked bool } @@ -69,7 +69,7 @@ func (n *NamespaceContainer) SetImplementation(impl support.NamespaceAccessImpl) n.impl = impl } -func (n *NamespaceContainer) getPusher(vers string) (resolve.Pusher, error) { +func (n *NamespaceContainer) getPusher(vers string) (regclient.Pusher, error) { err := n.assureCreated() if err != nil { return nil, err diff --git a/api/oci/extensions/repositories/ocireg/repository.go b/api/oci/extensions/repositories/ocireg/repository.go index bff1e5107..8d3110846 100644 --- a/api/oci/extensions/repositories/ocireg/repository.go +++ b/api/oci/extensions/repositories/ocireg/repository.go @@ -2,24 +2,19 @@ package ocireg import ( "context" - "crypto/tls" - "crypto/x509" - "net/http" "path" "strings" - "github.com/containerd/containerd/remotes/docker/config" "github.com/containerd/errdefs" "github.com/mandelsoft/goutils/errors" "github.com/mandelsoft/logging" + regconfig "github.com/regclient/regclient/config" "ocm.software/ocm/api/credentials" - "ocm.software/ocm/api/datacontext/attrs/rootcertsattr" "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/tech/docker" - "ocm.software/ocm/api/tech/docker/resolve" "ocm.software/ocm/api/tech/oci/identity" + "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/utils" ocmlog "ocm.software/ocm/api/utils/logging" "ocm.software/ocm/api/utils/refmgmt" @@ -114,7 +109,7 @@ func (r *RepositoryImpl) getCreds(comp string) (credentials.Credentials, error) return identity.GetCredentials(r.GetContext(), r.info.Locator, comp) } -func (r *RepositoryImpl) getResolver(comp string) (resolve.Resolver, error) { +func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { creds, err := r.getCreds(comp) if err != nil { if !errors.IsErrUnknownKind(err, credentials.KIND_CONSUMER) { @@ -126,57 +121,70 @@ func (r *RepositoryImpl) getResolver(comp string) (resolve.Resolver, error) { logger.Trace("no credentials") } - opts := docker.ResolverOptions{ - Hosts: docker.ConvertHosts(config.ConfigureHosts(context.Background(), config.HostOptions{ - UpdateClient: func(client *http.Client) error { - // copy from http.DefaultTransport with a roundtripper injection - client.Transport = ocmlog.NewRoundTripper(client.Transport, logger) - return nil - }, - Credentials: func(host string) (string, string, error) { - if creds != nil { - p := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) - if p == "" { - p = creds.GetProperty(credentials.ATTR_PASSWORD) - } - pw := "" - if p != "" { - pw = "***" - } - logger.Trace("query credentials", ocmlog.ATTR_USER, creds.GetProperty(credentials.ATTR_USERNAME), "pass", pw) - return creds.GetProperty(credentials.ATTR_USERNAME), p, nil - } - logger.Trace("no credentials") - return "", "", nil - }, - DefaultScheme: r.info.Scheme, - //nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version. - DefaultTLS: func() *tls.Config { - if r.info.Scheme == "http" { - return nil - } - return &tls.Config{ - // MinVersion: tls.VersionTLS13, - RootCAs: func() *x509.CertPool { - var rootCAs *x509.CertPool - if creds != nil { - c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY) - if c != "" { - rootCAs = x509.NewCertPool() - rootCAs.AppendCertsFromPEM([]byte(c)) - } - } - if rootCAs == nil { - rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true) - } - return rootCAs - }(), - } - }(), - })), + pass := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) + if pass == "" { + pass = creds.GetProperty(credentials.ATTR_PASSWORD) } - - return docker.NewResolver(opts), nil + username := creds.GetProperty(credentials.ATTR_USERNAME) + opts := regclient.ClientOptions{ + Host: ®config.Host{ + Name: "ghcr.io", + User: username, + Pass: pass, + }, + Version: comp, + } + //opts := docker.ResolverOptions{ + // Hosts: docker.ConvertHosts(config.ConfigureHosts(context.Background(), config.HostOptions{ + // UpdateClient: func(client *http.Client) error { + // // copy from http.DefaultTransport with a roundtripper injection + // client.Transport = ocmlog.NewRoundTripper(client.Transport, logger) + // return nil + // }, + // Credentials: func(host string) (string, string, error) { + // if creds != nil { + // p := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) + // if p == "" { + // p = creds.GetProperty(credentials.ATTR_PASSWORD) + // } + // pw := "" + // if p != "" { + // pw = "***" + // } + // logger.Trace("query credentials", ocmlog.ATTR_USER, creds.GetProperty(credentials.ATTR_USERNAME), "pass", pw) + // return creds.GetProperty(credentials.ATTR_USERNAME), p, nil + // } + // logger.Trace("no credentials") + // return "", "", nil + // }, + // DefaultScheme: r.info.Scheme, + // //nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version. + // DefaultTLS: func() *tls.Config { + // if r.info.Scheme == "http" { + // return nil + // } + // return &tls.Config{ + // // MinVersion: tls.VersionTLS13, + // RootCAs: func() *x509.CertPool { + // var rootCAs *x509.CertPool + // if creds != nil { + // c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY) + // if c != "" { + // rootCAs = x509.NewCertPool() + // rootCAs.AppendCertsFromPEM([]byte(c)) + // } + // } + // if rootCAs == nil { + // rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true) + // } + // return rootCAs + // }(), + // } + // }(), + // })), + //} + + return regclient.New(opts), nil } func (r *RepositoryImpl) GetRef(comp, vers string) string { diff --git a/api/oci/extensions/repositories/ocireg/utils.go b/api/oci/extensions/repositories/ocireg/utils.go index 17a96f040..76e98964b 100644 --- a/api/oci/extensions/repositories/ocireg/utils.go +++ b/api/oci/extensions/repositories/ocireg/utils.go @@ -11,10 +11,10 @@ import ( "github.com/containerd/log" "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" + "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/tech/docker/resolve" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" "ocm.software/ocm/api/utils/logging" @@ -81,12 +81,12 @@ func readAll(reader io.ReadCloser, err error) ([]byte, error) { return data, nil } -func push(ctx context.Context, p resolve.Pusher, blob blobaccess.BlobAccess) error { +func push(ctx context.Context, p regclient.Pusher, blob blobaccess.BlobAccess) error { desc := *artdesc.DefaultBlobDescriptor(blob) return pushData(ctx, p, desc, blob) } -func pushData(ctx context.Context, p resolve.Pusher, desc artdesc.Descriptor, data blobaccess.DataAccess) error { +func pushData(ctx context.Context, p regclient.Pusher, desc artdesc.Descriptor, data blobaccess.DataAccess) error { key := remotes.MakeRefKey(ctx, desc) if desc.Size == 0 { desc.Size = -1 @@ -100,8 +100,10 @@ func pushData(ctx context.Context, p resolve.Pusher, desc artdesc.Descriptor, da return nil } + return fmt.Errorf("failed to push: %w", err) } + return req.Commit(ctx, desc.Size, desc.Digest) } diff --git a/api/tech/docker/lister.go b/api/tech/docker/lister.go index efd3b8e1e..64845e27e 100644 --- a/api/tech/docker/lister.go +++ b/api/tech/docker/lister.go @@ -9,8 +9,7 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" "github.com/pkg/errors" - - "ocm.software/ocm/api/tech/docker/resolve" + "ocm.software/ocm/api/tech/regclient" ) var ErrObjectNotRequired = errors.New("object not required") @@ -24,7 +23,7 @@ type dockerLister struct { dockerBase *dockerBase } -func (r *dockerResolver) Lister(ctx context.Context, ref string) (resolve.Lister, error) { +func (r *dockerResolver) Lister(ctx context.Context, ref string) (regclient.Lister, error) { base, err := r.resolveDockerBase(ref) if err != nil { return nil, err diff --git a/api/tech/docker/pusher.go b/api/tech/docker/pusher.go index 708ad0f34..2c26f9b19 100644 --- a/api/tech/docker/pusher.go +++ b/api/tech/docker/pusher.go @@ -17,9 +17,9 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "ocm.software/ocm/api/tech/regclient" remoteserrors "ocm.software/ocm/api/tech/docker/errors" - "ocm.software/ocm/api/tech/docker/resolve" "ocm.software/ocm/api/utils/accessio" ) @@ -37,11 +37,11 @@ type dockerPusher struct { tracker StatusTracker } -func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor, src resolve.Source) (resolve.PushRequest, error) { +func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor, src regclient.Source) (regclient.PushRequest, error) { return p.push(ctx, desc, src, remotes.MakeRefKey(ctx, desc), false) } -func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, src resolve.Source, ref string, unavailableOnFail bool) (resolve.PushRequest, error) { +func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, src regclient.Source, ref string, unavailableOnFail bool) (regclient.PushRequest, error) { if l, ok := p.tracker.(StatusTrackLocker); ok { l.Lock(ref) defer l.Unlock(ref) @@ -322,7 +322,7 @@ type pushRequest struct { ref string responseC <-chan response - source resolve.Source + source regclient.Source isManifest bool expected digest.Digest diff --git a/api/tech/docker/resolver.go b/api/tech/docker/resolver.go index 292df03ae..8647ded6b 100644 --- a/api/tech/docker/resolver.go +++ b/api/tech/docker/resolver.go @@ -20,8 +20,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context/ctxhttp" + "ocm.software/ocm/api/tech/regclient" - "ocm.software/ocm/api/tech/docker/resolve" "ocm.software/ocm/api/utils/accessio" ) @@ -118,7 +118,7 @@ type dockerResolver struct { } // NewResolver returns a new resolver to a Docker registry. -func NewResolver(options ResolverOptions) resolve.Resolver { +func NewResolver(options ResolverOptions) regclient.Resolver { if options.Tracker == nil { options.Tracker = NewInMemoryTracker() } @@ -202,7 +202,7 @@ func (r *countingReader) Read(p []byte) (int, error) { return n, err } -var _ resolve.Resolver = &dockerResolver{} +var _ regclient.Resolver = &dockerResolver{} func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) { base, err := r.resolveDockerBase(ref) @@ -382,7 +382,7 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp return "", ocispec.Descriptor{}, firstErr } -func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (resolve.Fetcher, error) { +func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (regclient.Fetcher, error) { base, err := r.resolveDockerBase(ref) if err != nil { return nil, err @@ -393,7 +393,7 @@ func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (resolve.Fetch }, nil } -func (r *dockerResolver) Pusher(ctx context.Context, ref string) (resolve.Pusher, error) { +func (r *dockerResolver) Pusher(ctx context.Context, ref string) (regclient.Pusher, error) { base, err := r.resolveDockerBase(ref) if err != nil { return nil, err diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go new file mode 100644 index 000000000..d2c233c24 --- /dev/null +++ b/api/tech/oras/client.go @@ -0,0 +1 @@ +package oras diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go new file mode 100644 index 000000000..f709b05c2 --- /dev/null +++ b/api/tech/regclient/client.go @@ -0,0 +1,273 @@ +package regclient + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/opencontainers/go-digest" + ociv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient" + "github.com/regclient/regclient/config" + "github.com/regclient/regclient/scheme/reg" + "github.com/regclient/regclient/types/descriptor" + "github.com/regclient/regclient/types/manifest" + "github.com/regclient/regclient/types/platform" + regref "github.com/regclient/regclient/types/ref" +) + +type ClientOptions struct { + Host *config.Host + Version string +} + +type Client struct { + rc *regclient.RegClient + ref regref.Ref +} + +type pushRequest struct { + rc *regclient.RegClient + desc descriptor.Descriptor + ref regref.Ref +} + +// Commit and Status are actually not really used. Commit is a second stage operation and Status is never called in +// the library. +func (p *pushRequest) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { + return p.rc.Close(ctx, p.ref) +} + +func (p *pushRequest) Status() (content.Status, error) { + return content.Status{ + Ref: p.ref.Reference, + Total: p.desc.Size, + }, nil +} + +var _ PushRequest = &pushRequest{} + +var _ Resolver = &Client{} +var _ Fetcher = &Client{} +var _ Pusher = &Client{} +var _ Lister = &Client{} + +func New(opts ClientOptions) *Client { + rc := regclient.New( + regclient.WithConfigHost(*opts.Host), + regclient.WithDockerCerts(), + regclient.WithDockerCreds(), + regclient.WithUserAgent("containerd/"+opts.Version), + regclient.WithRegOpts( + //reg.WithCertDirs([]string{"."}), + reg.WithDelay(2*time.Second, 15*time.Second), + reg.WithRetryLimit(5), + reg.WithCache(5*time.Minute, 500), // built in cache!! Nice! + ), + ) + + return &Client{rc: rc} +} + +// Close must be called at the end of the operation. +func (c *Client) Close(ctx context.Context, ref regref.Ref) error { + return c.rc.Close(ctx, ref) +} + +func (c *Client) convertDescriptorToOCI(desc descriptor.Descriptor) ociv1.Descriptor { + var p *ociv1.Platform + if desc.Platform != nil { + p = &ociv1.Platform{ + Architecture: desc.Platform.Architecture, + OS: desc.Platform.OS, + OSVersion: desc.Platform.OSVersion, + OSFeatures: desc.Platform.OSFeatures, + Variant: desc.Platform.Variant, + } + } + + return ociv1.Descriptor{ + MediaType: desc.MediaType, + Size: desc.Size, + Digest: desc.Digest, + Platform: p, + URLs: desc.URLs, + Annotations: desc.Annotations, + Data: desc.Data, + ArtifactType: desc.ArtifactType, + } +} + +func (c *Client) convertDescriptorToRegClient(desc ociv1.Descriptor) descriptor.Descriptor { + var p *platform.Platform + if desc.Platform != nil { + p = &platform.Platform{ + Architecture: desc.Platform.Architecture, + OS: desc.Platform.OS, + OSVersion: desc.Platform.OSVersion, + OSFeatures: desc.Platform.OSFeatures, + Variant: desc.Platform.Variant, + } + } + + return descriptor.Descriptor{ + MediaType: desc.MediaType, + Size: desc.Size, + Digest: desc.Digest, + Platform: p, + URLs: desc.URLs, + Annotations: desc.Annotations, + Data: desc.Data, + ArtifactType: desc.ArtifactType, + } +} + +func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { + //TODO: figure out what to do about closing c.rc. + r, err := regref.New(ref) + if err != nil { + return "", ociv1.Descriptor{}, err + } + + // if digest is set it will use that. + m, err := c.rc.ManifestHead(ctx, r) + if err != nil { + if strings.Contains(err.Error(), "not found") { + // fallback to finding a blob if we have a digest + if r.Digest != "" { + blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ + Digest: digest.Digest(r.Digest), + Size: -1, + }) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return "", ociv1.Descriptor{}, errdefs.ErrNotFound + } + + return "", ociv1.Descriptor{}, err + } + + return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil + } + + return "", ociv1.Descriptor{}, errdefs.ErrNotFound + } + + return "", ociv1.Descriptor{}, fmt.Errorf("failed to get manifest: %w", err) + } + + return ref, c.convertDescriptorToOCI(m.GetDescriptor()), nil +} + +func (c *Client) Fetcher(ctx context.Context, ref string) (Fetcher, error) { + var err error + c.ref, err = regref.New(ref) + if err != nil { + return nil, err + } + + return c, nil +} + +func (c *Client) Pusher(ctx context.Context, ref string) (Pusher, error) { + var err error + c.ref, err = regref.New(ref) + if err != nil { + return nil, err + } + + return c, nil +} + +func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { + var err error + c.ref, err = regref.New(ref) + if err != nil { + return nil, err + } + + return c, nil +} + +func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (PushRequest, error) { + reader, err := src.Reader() + if err != nil { + return nil, err + } + + isManifest := false + switch d.MediaType { + case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, + ociv1.MediaTypeImageManifest, ociv1.MediaTypeImageIndex: + isManifest = true + } + if isManifest { + manifestContent, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + m, err := manifest.New(manifest.WithDesc(c.convertDescriptorToRegClient(d)), manifest.WithRef(c.ref), manifest.WithRaw(manifestContent)) + if err != nil { + return nil, fmt.Errorf("failed to create a manifest: %w", err) + } + + if err := c.rc.ManifestPut(ctx, c.ref, m); err != nil { + return nil, err + } + + return &pushRequest{ + desc: c.convertDescriptorToRegClient(d), + rc: c.rc, + ref: c.ref, + }, nil + } + + desc, err := c.rc.BlobPut(ctx, c.ref, c.convertDescriptorToRegClient(d), reader) + if err != nil { + return nil, err + } + + return &pushRequest{ + desc: desc, + rc: c.rc, + ref: c.ref, + }, nil +} + +func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadCloser, err error) { + defer func() { + if cerr := c.rc.Close(ctx, c.ref); cerr != nil { + err = errors.Join(err, fmt.Errorf("failed to close the client after fetch: %w", cerr)) + } + }() + + // set up closing the client after fetching is done. + reader, err := c.rc.BlobGet(ctx, c.ref, c.convertDescriptorToRegClient(desc)) + if err != nil { + return nil, err + } + + return reader, nil +} + +func (c *Client) List(ctx context.Context) (_ []string, err error) { + defer func() { + if cerr := c.rc.Close(ctx, c.ref); cerr != nil { + err = errors.Join(err, fmt.Errorf("failed to close the client after list: %w", cerr)) + } + }() + + tags, err := c.rc.TagList(ctx, c.ref) + if err != nil { + return nil, err + } + + return tags.Tags, nil +} diff --git a/api/tech/regclient/client_test.go b/api/tech/regclient/client_test.go new file mode 100644 index 000000000..0de7789ba --- /dev/null +++ b/api/tech/regclient/client_test.go @@ -0,0 +1,13 @@ +package regclient + +import ( + "testing" + + "github.com/regclient/regclient/config" + "github.com/stretchr/testify/require" +) + +func TestNew(t *testing.T) { + n := New(ClientOptions{Host: config.HostNew()}) + require.NotNil(t, n) +} diff --git a/api/tech/docker/resolve/interface.go b/api/tech/regclient/interface.go similarity index 99% rename from api/tech/docker/resolve/interface.go rename to api/tech/regclient/interface.go index 847600001..d11f0648d 100644 --- a/api/tech/docker/resolve/interface.go +++ b/api/tech/regclient/interface.go @@ -1,4 +1,4 @@ -package resolve +package regclient import ( "context" diff --git a/go.mod b/go.mod index 296f46db7..69cbffe6c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.23.2 replace github.com/spf13/cobra => github.com/open-component-model/cobra v0.0.0-20230329075350-b1fd876abfb9 +//replace github.com/regclient/regclient => /Users/skarlso/goprojects/regclient + require ( dario.cat/mergo v1.0.1 github.com/DataDog/gostackparse v0.7.0 @@ -56,6 +58,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 github.com/pkg/errors v0.9.1 github.com/redis/go-redis/v9 v9.7.0 + github.com/regclient/regclient v0.7.2 github.com/rogpeppe/go-internal v1.13.1 github.com/sigstore/cosign/v2 v2.4.1 github.com/sigstore/rekor v1.3.7 @@ -172,6 +175,7 @@ require ( github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/orderedmap v1.7.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect @@ -277,7 +281,6 @@ require ( github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/go.sum b/go.sum index 078bbf32c..8eecd224a 100644 --- a/go.sum +++ b/go.sum @@ -785,6 +785,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olareg/olareg v0.1.1 h1:Ui7q93zjcoF+U9U71sgqgZWByDoZOpqHitUXEu2xV+g= +github.com/olareg/olareg v0.1.1/go.mod h1:w8NP4SWrHHtxsFaUiv1lnCnYPm4sN1seCd2h7FK/dc0= github.com/oleiade/reflections v1.1.0 h1:D+I/UsXQB4esMathlt0kkZRJZdUDmhv5zGi/HOwYTWo= github.com/oleiade/reflections v1.1.0/go.mod h1:mCxx0QseeVCHs5Um5HhJeCKVC7AwS8kO67tky4rdisA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -874,6 +876,8 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.5.3 h1:kuvuJL/+MZIEdvtb/kTBRiRgY github.com/redis/go-redis/extra/redisotel/v9 v9.5.3/go.mod h1:7f/FMrf5RRRVHXgfk7CzSVzXHiWeuOQUu2bsVqWoa+g= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/regclient/regclient v0.7.2 h1:vcldDAwBMLtighYVMeb6qNt5+0hKg3AN2IkCc0JIJNM= +github.com/regclient/regclient v0.7.2/go.mod h1:QlA7W9/pvmbblOXM4d49JgfuOTwVXcUMKt3bFuOSVIQ= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= From 7dd651229b17db82dc971f48eea28928355eb4be Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:06:12 +0100 Subject: [PATCH 02/17] fix the nil and the unit test --- .../extensions/repositories/ocireg/blobs.go | 2 +- .../repositories/ocireg/namespace.go | 2 +- .../repositories/ocireg/repository.go | 20 ++++++++++----- .../extensions/repositories/ocireg/utils.go | 2 +- api/tech/regclient/client.go | 25 +++++++++++-------- 5 files changed, 32 insertions(+), 19 deletions(-) diff --git a/api/oci/extensions/repositories/ocireg/blobs.go b/api/oci/extensions/repositories/ocireg/blobs.go index d65f7e7dc..f92deba0a 100644 --- a/api/oci/extensions/repositories/ocireg/blobs.go +++ b/api/oci/extensions/repositories/ocireg/blobs.go @@ -7,10 +7,10 @@ import ( "github.com/mandelsoft/goutils/errors" "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" - "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/extensions/attrs/cacheattr" + "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" ) diff --git a/api/oci/extensions/repositories/ocireg/namespace.go b/api/oci/extensions/repositories/ocireg/namespace.go index 43477b011..cae5c05c3 100644 --- a/api/oci/extensions/repositories/ocireg/namespace.go +++ b/api/oci/extensions/repositories/ocireg/namespace.go @@ -7,12 +7,12 @@ import ( "github.com/containerd/errdefs" "github.com/mandelsoft/goutils/errors" "github.com/opencontainers/go-digest" - "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/cpi/support" "ocm.software/ocm/api/oci/extensions/actions/oci-repository-prepare" + "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" "ocm.software/ocm/api/utils/logging" diff --git a/api/oci/extensions/repositories/ocireg/repository.go b/api/oci/extensions/repositories/ocireg/repository.go index 8d3110846..68bdfa883 100644 --- a/api/oci/extensions/repositories/ocireg/repository.go +++ b/api/oci/extensions/repositories/ocireg/repository.go @@ -121,16 +121,24 @@ func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { logger.Trace("no credentials") } - pass := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) - if pass == "" { - pass = creds.GetProperty(credentials.ATTR_PASSWORD) + var ( + password string + username string + ) + + if creds != nil { + password = creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) + if password == "" { + password = creds.GetProperty(credentials.ATTR_PASSWORD) + } + username = creds.GetProperty(credentials.ATTR_USERNAME) } - username := creds.GetProperty(credentials.ATTR_USERNAME) + opts := regclient.ClientOptions{ Host: ®config.Host{ - Name: "ghcr.io", + Name: "ghcr.io", //TODO: Need to figure out how to set the host. User: username, - Pass: pass, + Pass: password, }, Version: comp, } diff --git a/api/oci/extensions/repositories/ocireg/utils.go b/api/oci/extensions/repositories/ocireg/utils.go index 76e98964b..8c58c5237 100644 --- a/api/oci/extensions/repositories/ocireg/utils.go +++ b/api/oci/extensions/repositories/ocireg/utils.go @@ -11,10 +11,10 @@ import ( "github.com/containerd/log" "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" - "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" + "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" "ocm.software/ocm/api/utils/logging" diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index f709b05c2..5ba1d7486 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -53,10 +53,12 @@ func (p *pushRequest) Status() (content.Status, error) { var _ PushRequest = &pushRequest{} -var _ Resolver = &Client{} -var _ Fetcher = &Client{} -var _ Pusher = &Client{} -var _ Lister = &Client{} +var ( + _ Resolver = &Client{} + _ Fetcher = &Client{} + _ Pusher = &Client{} + _ Lister = &Client{} +) func New(opts ClientOptions) *Client { rc := regclient.New( @@ -65,7 +67,7 @@ func New(opts ClientOptions) *Client { regclient.WithDockerCreds(), regclient.WithUserAgent("containerd/"+opts.Version), regclient.WithRegOpts( - //reg.WithCertDirs([]string{"."}), + // reg.WithCertDirs([]string{"."}), reg.WithDelay(2*time.Second, 15*time.Second), reg.WithRetryLimit(5), reg.WithCache(5*time.Minute, 500), // built in cache!! Nice! @@ -129,7 +131,7 @@ func (c *Client) convertDescriptorToRegClient(desc ociv1.Descriptor) descriptor. } func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { - //TODO: figure out what to do about closing c.rc. + // TODO: figure out what to do about closing c.rc. r, err := regref.New(ref) if err != nil { return "", ociv1.Descriptor{}, err @@ -141,16 +143,15 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip if strings.Contains(err.Error(), "not found") { // fallback to finding a blob if we have a digest if r.Digest != "" { - blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ + blob, err := c.rc.BlobGet(ctx, r, descriptor.Descriptor{ Digest: digest.Digest(r.Digest), - Size: -1, }) if err != nil { if strings.Contains(err.Error(), "not found") { return "", ociv1.Descriptor{}, errdefs.ErrNotFound } - return "", ociv1.Descriptor{}, err + return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob head: %w", err) } return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil @@ -249,9 +250,13 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo }() // set up closing the client after fetching is done. + // -1 is not a thing in regclient. + if desc.Size < 0 { + desc.Size = 0 + } reader, err := c.rc.BlobGet(ctx, c.ref, c.convertDescriptorToRegClient(desc)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get the blob reader: %w", err) } return reader, nil From 1b51dc4c1b62e0c0992be059a35aabdcd18180f0 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:13:01 +0100 Subject: [PATCH 03/17] updated internal ref and the way we resolve manifests --- api/tech/regclient/client.go | 39 ++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index 5ba1d7486..ab357bab9 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -1,6 +1,7 @@ package regclient import ( + "bytes" "context" "errors" "fmt" @@ -163,6 +164,9 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip return "", ociv1.Descriptor{}, fmt.Errorf("failed to get manifest: %w", err) } + // update the Ref of the client to the resolved reference. + c.ref = r + return ref, c.convertDescriptorToOCI(m.GetDescriptor()), nil } @@ -202,13 +206,7 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (Push return nil, err } - isManifest := false - switch d.MediaType { - case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, - ociv1.MediaTypeImageManifest, ociv1.MediaTypeImageIndex: - isManifest = true - } - if isManifest { + if c.isManifest(d) { manifestContent, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("failed to read manifest: %w", err) @@ -223,6 +221,7 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (Push return nil, err } + // pushRequest closes the RC on `Commit`. return &pushRequest{ desc: c.convertDescriptorToRegClient(d), rc: c.rc, @@ -249,11 +248,25 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo } }() - // set up closing the client after fetching is done. // -1 is not a thing in regclient. if desc.Size < 0 { desc.Size = 0 } + + if c.isManifest(desc) { + manifestContent, err := c.rc.ManifestGet(ctx, c.ref) + if err != nil { + return nil, err + } + + body, err := manifestContent.RawBody() + if err != nil { + return nil, err + } + + return io.NopCloser(bytes.NewReader(body)), nil + } + reader, err := c.rc.BlobGet(ctx, c.ref, c.convertDescriptorToRegClient(desc)) if err != nil { return nil, fmt.Errorf("failed to get the blob reader: %w", err) @@ -276,3 +289,13 @@ func (c *Client) List(ctx context.Context) (_ []string, err error) { return tags.Tags, nil } + +func (c *Client) isManifest(desc ociv1.Descriptor) bool { + switch desc.MediaType { + case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, + ociv1.MediaTypeImageManifest, ociv1.MediaTypeImageIndex: + return true + } + + return false +} From cc7c285fd9db20d777a89c2d489f32fb6c7e12c5 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:03:16 +0100 Subject: [PATCH 04/17] fix fetching --- api/oci/artdesc/index.go | 2 + api/oci/cpi/state.go | 2 + api/tech/regclient/client.go | 41 +++++++++++-------- api/utils/accessobj/accessstate.go | 1 + .../tour/07-resource-management/example.go | 15 +++++++ hack/generate.sh | 2 +- 6 files changed, 46 insertions(+), 17 deletions(-) diff --git a/api/oci/artdesc/index.go b/api/oci/artdesc/index.go index fe741236a..66e39a2ca 100644 --- a/api/oci/artdesc/index.go +++ b/api/oci/artdesc/index.go @@ -2,6 +2,7 @@ package artdesc import ( "encoding/json" + "fmt" "github.com/mandelsoft/goutils/errors" "github.com/opencontainers/go-digest" @@ -104,6 +105,7 @@ func (i *Index) AddManifest(d *Descriptor) { //////////////////////////////////////////////////////////////////////////////// func DecodeIndex(data []byte) (*Index, error) { + fmt.Println("index handler") var d Index if err := json.Unmarshal(data, &d); err != nil { diff --git a/api/oci/cpi/state.go b/api/oci/cpi/state.go index 53abaf18a..774727b05 100644 --- a/api/oci/cpi/state.go +++ b/api/oci/cpi/state.go @@ -1,6 +1,7 @@ package cpi import ( + "fmt" "reflect" "ocm.software/ocm/api/oci/artdesc" @@ -76,6 +77,7 @@ func (i ArtifactStateHandler) Encode(d interface{}) ([]byte, error) { } func (i ArtifactStateHandler) Decode(data []byte) (interface{}, error) { + fmt.Println("artifact state handler") return artdesc.Decode(data) } diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index ab357bab9..42e2959cc 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -138,26 +138,31 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip return "", ociv1.Descriptor{}, err } + if r.Digest != "" { + blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ + Digest: digest.Digest(r.Digest), + }) + defer blob.Close() // we can safely close it as this is not when we read it. + + if err != nil { + if strings.Contains(err.Error(), "not found") { + return "", ociv1.Descriptor{}, errdefs.ErrNotFound + } + + return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob head: %w", err) + } + + c.ref = r + + return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil + } + // if digest is set it will use that. + fmt.Println("we are in manifest") m, err := c.rc.ManifestHead(ctx, r) if err != nil { + fmt.Println("we are in manifest error: ", err) if strings.Contains(err.Error(), "not found") { - // fallback to finding a blob if we have a digest - if r.Digest != "" { - blob, err := c.rc.BlobGet(ctx, r, descriptor.Descriptor{ - Digest: digest.Digest(r.Digest), - }) - if err != nil { - if strings.Contains(err.Error(), "not found") { - return "", ociv1.Descriptor{}, errdefs.ErrNotFound - } - - return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob head: %w", err) - } - - return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil - } - return "", ociv1.Descriptor{}, errdefs.ErrNotFound } @@ -167,6 +172,7 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip // update the Ref of the client to the resolved reference. c.ref = r + fmt.Println("we returned") return ref, c.convertDescriptorToOCI(m.GetDescriptor()), nil } @@ -253,7 +259,10 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo desc.Size = 0 } + fmt.Println("in fetch: ", desc, c.ref) + if c.isManifest(desc) { + fmt.Println("in manifest: ", desc, c.ref) manifestContent, err := c.rc.ManifestGet(ctx, c.ref) if err != nil { return nil, err diff --git a/api/utils/accessobj/accessstate.go b/api/utils/accessobj/accessstate.go index a53e49430..bb0d35ee5 100644 --- a/api/utils/accessobj/accessstate.go +++ b/api/utils/accessobj/accessstate.go @@ -153,6 +153,7 @@ func newState(mode AccessMode, a StateAccess, p StateHandler) (*state, error) { return nil, fmt.Errorf("failed to get blob data: %w", err) } + fmt.Println("data: ", blob.MimeType()) blob = blobaccess.ForData(blob.MimeType(), data) // cache original data current, err = p.Decode(data) if err != nil { diff --git a/examples/lib/tour/07-resource-management/example.go b/examples/lib/tour/07-resource-management/example.go index cffd10c17..ccd3471e8 100644 --- a/examples/lib/tour/07-resource-management/example.go +++ b/examples/lib/tour/07-resource-management/example.go @@ -160,6 +160,7 @@ func HandleResources(resources []Resource) error { // --- end handle --- func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, error) { + fmt.Println("BEGIN GATHERING") var resources []Resource spec := ocireg.NewRepositorySpec("ghcr.io/open-component-model/ocm") @@ -172,6 +173,7 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro if err != nil { return nil, errors.Wrapf(err, "cannot setup repository") } + fmt.Println("GOT REPO:", spec) // to release potentially allocated temporary resources, // many objects must be closed, if they should not be used @@ -185,11 +187,15 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro // All kinds of repositories, regardless of their type // feature the same interface to work with OCM content. // --- begin lookup component --- + fmt.Println("Looking up component...") c, err := repo.LookupComponent("ocm.software/ocmcli") if err != nil { return nil, errors.Wrapf(err, "cannot lookup component") } defer c.Close() + + fmt.Println("Lookued up Component: ", c) + // --- end lookup component --- // Now we look for the versions of the component @@ -199,6 +205,8 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro return nil, errors.Wrapf(err, "cannot query version names") } + fmt.Println("Versions:", versions) + // OCM version names must follow the SemVer rules. // Therefore, we can simply order the versions and print them. err = semverutils.SortVersions(versions) @@ -213,6 +221,8 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro // --- begin lookup version --- // to retrieve the latest version use // cv, err := c.LookupVersion(versions[len(versions)-1]) + + fmt.Println("looking up component version: v0.17.0") cv, err := c.LookupVersion("0.17.0") if err != nil { return nil, errors.Wrapf(err, "cannot get latest version") @@ -232,25 +242,30 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro // like the repository specification. // --- begin resources --- for _, r := range cv.GetResources() { + fmt.Println("trying to fetch resource: ", r.Meta()) res := factory.Create( r.Meta().GetIdentity(cv.GetDescriptor().Resources), r.Meta().GetType(), ) acc, err := r.Access() if err != nil { + fmt.Printf("cannot access resource 1: %s\n", err) res.SetError(err.Error()) } else { m, err := acc.AccessMethod(cv) + fmt.Printf("cannot access resource 2: %s\n", err) if err == nil { // delegate data handling to target // we don't know, how this is implemented. err = res.AddDataFromMethod(ctx, m) + fmt.Println("ERROR on 3: ", err, m) if err != nil { res.SetError(err.Error()) } // release local usage of the access method object m.Close() } else { + fmt.Printf("cannot access resource 3: %s\n", err) res.SetError(err.Error()) } } diff --git a/hack/generate.sh b/hack/generate.sh index d574eb582..7e6d3cc72 100755 --- a/hack/generate.sh +++ b/hack/generate.sh @@ -4,4 +4,4 @@ set -e echo "> Generate" -GO111MODULE=on go generate -mod=mod $@ \ No newline at end of file +GO111MODULE=on go generate -v -mod=mod $@ From cd8281bc7fef98fecd91368c128f7b5b0418796e Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:40:12 +0100 Subject: [PATCH 05/17] fix fetching --- .../repositories/ocireg/repository.go | 10 ++-- api/tech/regclient/client.go | 58 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/api/oci/extensions/repositories/ocireg/repository.go b/api/oci/extensions/repositories/ocireg/repository.go index 68bdfa883..3183f085d 100644 --- a/api/oci/extensions/repositories/ocireg/repository.go +++ b/api/oci/extensions/repositories/ocireg/repository.go @@ -135,10 +135,12 @@ func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { } opts := regclient.ClientOptions{ - Host: ®config.Host{ - Name: "ghcr.io", //TODO: Need to figure out how to set the host. - User: username, - Pass: password, + Host: []regconfig.Host{ + { + Name: "ghcr.io", //TODO: Need to figure out how to set the host. + User: username, + Pass: password, + }, }, Version: comp, } diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index 42e2959cc..d3721f0f9 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "strings" "time" "github.com/containerd/containerd/content" @@ -18,13 +17,15 @@ import ( "github.com/regclient/regclient/config" "github.com/regclient/regclient/scheme/reg" "github.com/regclient/regclient/types/descriptor" + regerr "github.com/regclient/regclient/types/errs" "github.com/regclient/regclient/types/manifest" "github.com/regclient/regclient/types/platform" regref "github.com/regclient/regclient/types/ref" ) type ClientOptions struct { - Host *config.Host + // TODO: add multiple hosts parsing + Host []config.Host Version string } @@ -63,7 +64,7 @@ var ( func New(opts ClientOptions) *Client { rc := regclient.New( - regclient.WithConfigHost(*opts.Host), + regclient.WithConfigHost(opts.Host...), regclient.WithDockerCerts(), regclient.WithDockerCreds(), regclient.WithUserAgent("containerd/"+opts.Version), @@ -138,32 +139,30 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip return "", ociv1.Descriptor{}, err } - if r.Digest != "" { - blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ - Digest: digest.Digest(r.Digest), - }) - defer blob.Close() // we can safely close it as this is not when we read it. - - if err != nil { - if strings.Contains(err.Error(), "not found") { - return "", ociv1.Descriptor{}, errdefs.ErrNotFound - } - - return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob head: %w", err) - } - - c.ref = r - - return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil - } - - // if digest is set it will use that. - fmt.Println("we are in manifest") - m, err := c.rc.ManifestHead(ctx, r) + // first, try to find the manifest + m, err := c.rc.ManifestGet(ctx, r) if err != nil { - fmt.Println("we are in manifest error: ", err) - if strings.Contains(err.Error(), "not found") { - return "", ociv1.Descriptor{}, errdefs.ErrNotFound + if errors.Is(err, regerr.ErrNotFound) { + // try to find a blob + if r.Digest != "" { + blob, err := c.rc.BlobGet(ctx, r, descriptor.Descriptor{ + Digest: digest.Digest(r.Digest), + }) + defer blob.Close() // we can safely close it as this is not when we read it. + + if err != nil { + if errors.Is(err, regerr.ErrNotFound) { + return "", ociv1.Descriptor{}, errdefs.ErrNotFound + } + + return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob head: %w", err) + } + + // update the reference that has been resolved successfully + c.ref = r + + return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil + } } return "", ociv1.Descriptor{}, fmt.Errorf("failed to get manifest: %w", err) @@ -172,7 +171,6 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip // update the Ref of the client to the resolved reference. c.ref = r - fmt.Println("we returned") return ref, c.convertDescriptorToOCI(m.GetDescriptor()), nil } @@ -224,7 +222,7 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (Push } if err := c.rc.ManifestPut(ctx, c.ref, m); err != nil { - return nil, err + return nil, fmt.Errorf("failed to put manifest: %w", err) } // pushRequest closes the RC on `Commit`. From 8ad52af3bc175b0e35657beb38d2e84a18135ef3 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:02:44 +0100 Subject: [PATCH 06/17] using heads --- api/tech/docker/fetcher.go | 4 +++- api/tech/regclient/client.go | 8 +++++--- examples/lib/tour/07-resource-management/example.go | 4 ---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/tech/docker/fetcher.go b/api/tech/docker/fetcher.go index 4a2eec584..262c9fa6b 100644 --- a/api/tech/docker/fetcher.go +++ b/api/tech/docker/fetcher.go @@ -25,6 +25,8 @@ type dockerFetcher struct { func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { ctx = log.WithLogger(ctx, log.G(ctx).WithField("digest", desc.Digest)) + fmt.Println("Fetching: ", r.refspec, desc) + hosts := r.filterHosts(HostCapabilityPull) if len(hosts) == 0 { return nil, errors.Wrap(errdefs.ErrNotFound, "no pull hosts") @@ -39,7 +41,7 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R // firstly try fetch via external urls for _, us := range desc.URLs { ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", us)) - + fmt.Println("us: ", r.refspec, desc, us) u, err := url.Parse(us) if err != nil { log.G(ctx).WithError(err).Debug("failed to parse") diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index d3721f0f9..43963dfbd 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -140,12 +140,12 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip } // first, try to find the manifest - m, err := c.rc.ManifestGet(ctx, r) + m, err := c.rc.ManifestHead(ctx, r) if err != nil { if errors.Is(err, regerr.ErrNotFound) { // try to find a blob if r.Digest != "" { - blob, err := c.rc.BlobGet(ctx, r, descriptor.Descriptor{ + blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ Digest: digest.Digest(r.Digest), }) defer blob.Close() // we can safely close it as this is not when we read it. @@ -261,7 +261,7 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo if c.isManifest(desc) { fmt.Println("in manifest: ", desc, c.ref) - manifestContent, err := c.rc.ManifestGet(ctx, c.ref) + manifestContent, err := c.rc.ManifestGet(ctx, c.ref, regclient.WithManifestDesc(c.convertDescriptorToRegClient(desc))) if err != nil { return nil, err } @@ -274,6 +274,8 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo return io.NopCloser(bytes.NewReader(body)), nil } + fmt.Println("in blob get: ", desc, c.ref) + reader, err := c.rc.BlobGet(ctx, c.ref, c.convertDescriptorToRegClient(desc)) if err != nil { return nil, fmt.Errorf("failed to get the blob reader: %w", err) diff --git a/examples/lib/tour/07-resource-management/example.go b/examples/lib/tour/07-resource-management/example.go index ccd3471e8..3734c6585 100644 --- a/examples/lib/tour/07-resource-management/example.go +++ b/examples/lib/tour/07-resource-management/example.go @@ -249,23 +249,19 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro ) acc, err := r.Access() if err != nil { - fmt.Printf("cannot access resource 1: %s\n", err) res.SetError(err.Error()) } else { m, err := acc.AccessMethod(cv) - fmt.Printf("cannot access resource 2: %s\n", err) if err == nil { // delegate data handling to target // we don't know, how this is implemented. err = res.AddDataFromMethod(ctx, m) - fmt.Println("ERROR on 3: ", err, m) if err != nil { res.SetError(err.Error()) } // release local usage of the access method object m.Close() } else { - fmt.Printf("cannot access resource 3: %s\n", err) res.SetError(err.Error()) } } From a51766cdccd92d4f5c9e3e15e3cc57ab77b159e0 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:45:55 +0100 Subject: [PATCH 07/17] added delayed reader and blob head check --- api/oci/artdesc/index.go | 2 - api/oci/cpi/state.go | 2 - api/tech/docker/fetcher.go | 3 - api/tech/regclient/client.go | 31 ++++++---- api/tech/regclient/client_test.go | 5 +- api/tech/regclient/delayed_reader.go | 56 +++++++++++++++++++ api/utils/accessobj/accessstate.go | 1 - .../tour/07-resource-management/example.go | 11 ---- hack/generate.sh | 2 +- 9 files changed, 80 insertions(+), 33 deletions(-) create mode 100644 api/tech/regclient/delayed_reader.go diff --git a/api/oci/artdesc/index.go b/api/oci/artdesc/index.go index 66e39a2ca..fe741236a 100644 --- a/api/oci/artdesc/index.go +++ b/api/oci/artdesc/index.go @@ -2,7 +2,6 @@ package artdesc import ( "encoding/json" - "fmt" "github.com/mandelsoft/goutils/errors" "github.com/opencontainers/go-digest" @@ -105,7 +104,6 @@ func (i *Index) AddManifest(d *Descriptor) { //////////////////////////////////////////////////////////////////////////////// func DecodeIndex(data []byte) (*Index, error) { - fmt.Println("index handler") var d Index if err := json.Unmarshal(data, &d); err != nil { diff --git a/api/oci/cpi/state.go b/api/oci/cpi/state.go index 774727b05..53abaf18a 100644 --- a/api/oci/cpi/state.go +++ b/api/oci/cpi/state.go @@ -1,7 +1,6 @@ package cpi import ( - "fmt" "reflect" "ocm.software/ocm/api/oci/artdesc" @@ -77,7 +76,6 @@ func (i ArtifactStateHandler) Encode(d interface{}) ([]byte, error) { } func (i ArtifactStateHandler) Decode(data []byte) (interface{}, error) { - fmt.Println("artifact state handler") return artdesc.Decode(data) } diff --git a/api/tech/docker/fetcher.go b/api/tech/docker/fetcher.go index 262c9fa6b..194efe93e 100644 --- a/api/tech/docker/fetcher.go +++ b/api/tech/docker/fetcher.go @@ -25,8 +25,6 @@ type dockerFetcher struct { func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { ctx = log.WithLogger(ctx, log.G(ctx).WithField("digest", desc.Digest)) - fmt.Println("Fetching: ", r.refspec, desc) - hosts := r.filterHosts(HostCapabilityPull) if len(hosts) == 0 { return nil, errors.Wrap(errdefs.ErrNotFound, "no pull hosts") @@ -41,7 +39,6 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R // firstly try fetch via external urls for _, us := range desc.URLs { ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", us)) - fmt.Println("us: ", r.refspec, desc, us) u, err := url.Parse(us) if err != nil { log.G(ctx).WithError(err).Debug("failed to parse") diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index 43963dfbd..88c187075 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -24,7 +24,6 @@ import ( ) type ClientOptions struct { - // TODO: add multiple hosts parsing Host []config.Host Version string } @@ -40,8 +39,9 @@ type pushRequest struct { ref regref.Ref } -// Commit and Status are actually not really used. Commit is a second stage operation and Status is never called in -// the library. +// Commit and Status are actually not really used in the library. Commit is a second stage operation and Status is never called in +// the library. The Status was a thing mostly in docker being used to track chunk reads. But that's taken care of +// by regclient internally. func (p *pushRequest) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { return p.rc.Close(ctx, p.ref) } @@ -133,7 +133,6 @@ func (c *Client) convertDescriptorToRegClient(desc ociv1.Descriptor) descriptor. } func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { - // TODO: figure out what to do about closing c.rc. r, err := regref.New(ref) if err != nil { return "", ociv1.Descriptor{}, err @@ -257,12 +256,13 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo desc.Size = 0 } - fmt.Println("in fetch: ", desc, c.ref) - if c.isManifest(desc) { - fmt.Println("in manifest: ", desc, c.ref) manifestContent, err := c.rc.ManifestGet(ctx, c.ref, regclient.WithManifestDesc(c.convertDescriptorToRegClient(desc))) if err != nil { + if errors.Is(err, regerr.ErrNotFound) { + return nil, errdefs.ErrNotFound + } + return nil, err } @@ -274,14 +274,21 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo return io.NopCloser(bytes.NewReader(body)), nil } - fmt.Println("in blob get: ", desc, c.ref) - - reader, err := c.rc.BlobGet(ctx, c.ref, c.convertDescriptorToRegClient(desc)) + // check if the blob exists so we can bail early + _, err = c.rc.BlobHead(ctx, c.ref, c.convertDescriptorToRegClient(desc)) if err != nil { - return nil, fmt.Errorf("failed to get the blob reader: %w", err) + if errors.Is(err, regerr.ErrNotFound) { + return nil, errdefs.ErrNotFound + } + + return nil, err + } + + delayer := func() (io.ReadCloser, error) { + return c.rc.BlobGet(ctx, c.ref, c.convertDescriptorToRegClient(desc)) } - return reader, nil + return newDelayedReader(delayer) } func (c *Client) List(ctx context.Context) (_ []string, err error) { diff --git a/api/tech/regclient/client_test.go b/api/tech/regclient/client_test.go index 0de7789ba..b5045d702 100644 --- a/api/tech/regclient/client_test.go +++ b/api/tech/regclient/client_test.go @@ -8,6 +8,9 @@ import ( ) func TestNew(t *testing.T) { - n := New(ClientOptions{Host: config.HostNew()}) + host := config.HostNew() + n := New(ClientOptions{Host: []config.Host{ + *host, + }}) require.NotNil(t, n) } diff --git a/api/tech/regclient/delayed_reader.go b/api/tech/regclient/delayed_reader.go new file mode 100644 index 000000000..19fb9f220 --- /dev/null +++ b/api/tech/regclient/delayed_reader.go @@ -0,0 +1,56 @@ +package regclient + +import "io" + +// delayedReader sets up a reader that only fetches a blob +// upon explicit reading request, otherwise, it stores the +// way of getting the reader. +type delayedReader struct { + open func() (io.ReadCloser, error) + rc io.ReadCloser + closed bool +} + +func newDelayedReader(open func() (io.ReadCloser, error)) (io.ReadCloser, error) { + return &delayedReader{ + open: open, + }, nil +} + +func (d *delayedReader) Read(p []byte) (n int, err error) { + if d.closed { + return 0, io.EOF + } + + reader, err := d.reader() + if err != nil { + return 0, err + + } + + return reader.Read(p) +} + +func (d *delayedReader) reader() (io.ReadCloser, error) { + if d.rc != nil { + return d.rc, nil + } + + rc, err := d.open() + if err != nil { + return nil, err + } + + d.rc = rc + return rc, nil +} + +func (d *delayedReader) Close() error { + if d.closed { + return nil + } + + // we close regardless of an error + d.closed = true + return d.rc.Close() +} diff --git a/api/utils/accessobj/accessstate.go b/api/utils/accessobj/accessstate.go index bb0d35ee5..a53e49430 100644 --- a/api/utils/accessobj/accessstate.go +++ b/api/utils/accessobj/accessstate.go @@ -153,7 +153,6 @@ func newState(mode AccessMode, a StateAccess, p StateHandler) (*state, error) { return nil, fmt.Errorf("failed to get blob data: %w", err) } - fmt.Println("data: ", blob.MimeType()) blob = blobaccess.ForData(blob.MimeType(), data) // cache original data current, err = p.Decode(data) if err != nil { diff --git a/examples/lib/tour/07-resource-management/example.go b/examples/lib/tour/07-resource-management/example.go index 3734c6585..cffd10c17 100644 --- a/examples/lib/tour/07-resource-management/example.go +++ b/examples/lib/tour/07-resource-management/example.go @@ -160,7 +160,6 @@ func HandleResources(resources []Resource) error { // --- end handle --- func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, error) { - fmt.Println("BEGIN GATHERING") var resources []Resource spec := ocireg.NewRepositorySpec("ghcr.io/open-component-model/ocm") @@ -173,7 +172,6 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro if err != nil { return nil, errors.Wrapf(err, "cannot setup repository") } - fmt.Println("GOT REPO:", spec) // to release potentially allocated temporary resources, // many objects must be closed, if they should not be used @@ -187,15 +185,11 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro // All kinds of repositories, regardless of their type // feature the same interface to work with OCM content. // --- begin lookup component --- - fmt.Println("Looking up component...") c, err := repo.LookupComponent("ocm.software/ocmcli") if err != nil { return nil, errors.Wrapf(err, "cannot lookup component") } defer c.Close() - - fmt.Println("Lookued up Component: ", c) - // --- end lookup component --- // Now we look for the versions of the component @@ -205,8 +199,6 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro return nil, errors.Wrapf(err, "cannot query version names") } - fmt.Println("Versions:", versions) - // OCM version names must follow the SemVer rules. // Therefore, we can simply order the versions and print them. err = semverutils.SortVersions(versions) @@ -221,8 +213,6 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro // --- begin lookup version --- // to retrieve the latest version use // cv, err := c.LookupVersion(versions[len(versions)-1]) - - fmt.Println("looking up component version: v0.17.0") cv, err := c.LookupVersion("0.17.0") if err != nil { return nil, errors.Wrapf(err, "cannot get latest version") @@ -242,7 +232,6 @@ func GatherResources(ctx ocm.Context, factory ResourceFactory) ([]Resource, erro // like the repository specification. // --- begin resources --- for _, r := range cv.GetResources() { - fmt.Println("trying to fetch resource: ", r.Meta()) res := factory.Create( r.Meta().GetIdentity(cv.GetDescriptor().Resources), r.Meta().GetType(), diff --git a/hack/generate.sh b/hack/generate.sh index 7e6d3cc72..d2e986498 100755 --- a/hack/generate.sh +++ b/hack/generate.sh @@ -4,4 +4,4 @@ set -e echo "> Generate" -GO111MODULE=on go generate -v -mod=mod $@ +GO111MODULE=on go generate -mod=mod $@ From b3911c9d54c47681fd3d46e1863a2b79ad313c04 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:54:11 +0100 Subject: [PATCH 08/17] add deprecation tags on docker code and add returning not found in client --- api/tech/docker/README.md | 2 ++ api/tech/docker/errors/errors.go | 2 ++ api/tech/docker/fetcher.go | 2 ++ api/tech/docker/handler.go | 2 ++ api/tech/docker/httpreadseeker.go | 2 ++ api/tech/docker/lister.go | 2 ++ api/tech/docker/orig.go | 2 ++ api/tech/docker/pusher.go | 2 ++ api/tech/docker/registry.go | 2 ++ api/tech/docker/resolver.go | 2 ++ api/tech/regclient/client.go | 15 +++++++++++---- 11 files changed, 31 insertions(+), 4 deletions(-) diff --git a/api/tech/docker/README.md b/api/tech/docker/README.md index 096a9c1e1..7a6a1ae7f 100644 --- a/api/tech/docker/README.md +++ b/api/tech/docker/README.md @@ -2,3 +2,5 @@ Taken from github.com/containerd/containerd remotes/docker to add list endpoints Fix retry of requests with ResendBuffer + +This code is deprecated in favour of `regclient` package. diff --git a/api/tech/docker/errors/errors.go b/api/tech/docker/errors/errors.go index a158f75b5..71575a5da 100644 --- a/api/tech/docker/errors/errors.go +++ b/api/tech/docker/errors/errors.go @@ -1,3 +1,5 @@ +//go:build deprecated + /* Copyright The containerd Authors. diff --git a/api/tech/docker/fetcher.go b/api/tech/docker/fetcher.go index 194efe93e..4b325adac 100644 --- a/api/tech/docker/fetcher.go +++ b/api/tech/docker/fetcher.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/docker/handler.go b/api/tech/docker/handler.go index 0ff9959ad..c761d6d0d 100644 --- a/api/tech/docker/handler.go +++ b/api/tech/docker/handler.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/docker/httpreadseeker.go b/api/tech/docker/httpreadseeker.go index c6b803810..374919b8d 100644 --- a/api/tech/docker/httpreadseeker.go +++ b/api/tech/docker/httpreadseeker.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/docker/lister.go b/api/tech/docker/lister.go index 64845e27e..95b7a5117 100644 --- a/api/tech/docker/lister.go +++ b/api/tech/docker/lister.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/docker/orig.go b/api/tech/docker/orig.go index c9b2468fb..5ed0006d6 100644 --- a/api/tech/docker/orig.go +++ b/api/tech/docker/orig.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/docker/pusher.go b/api/tech/docker/pusher.go index 2c26f9b19..53b2a785f 100644 --- a/api/tech/docker/pusher.go +++ b/api/tech/docker/pusher.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/docker/registry.go b/api/tech/docker/registry.go index 795dd6e24..90ed31483 100644 --- a/api/tech/docker/registry.go +++ b/api/tech/docker/registry.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/docker/resolver.go b/api/tech/docker/resolver.go index 8647ded6b..a5b5f3161 100644 --- a/api/tech/docker/resolver.go +++ b/api/tech/docker/resolver.go @@ -1,3 +1,5 @@ +//go:build deprecated + package docker import ( diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index 88c187075..c6fc2d8be 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "strings" "time" "github.com/containerd/containerd/content" @@ -141,7 +142,7 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip // first, try to find the manifest m, err := c.rc.ManifestHead(ctx, r) if err != nil { - if errors.Is(err, regerr.ErrNotFound) { + if c.isNotFoundError(err) { // try to find a blob if r.Digest != "" { blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ @@ -150,7 +151,7 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip defer blob.Close() // we can safely close it as this is not when we read it. if err != nil { - if errors.Is(err, regerr.ErrNotFound) { + if c.isNotFoundError(err) { return "", ociv1.Descriptor{}, errdefs.ErrNotFound } @@ -162,6 +163,8 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil } + + return "", ociv1.Descriptor{}, errdefs.ErrNotFound } return "", ociv1.Descriptor{}, fmt.Errorf("failed to get manifest: %w", err) @@ -259,7 +262,7 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo if c.isManifest(desc) { manifestContent, err := c.rc.ManifestGet(ctx, c.ref, regclient.WithManifestDesc(c.convertDescriptorToRegClient(desc))) if err != nil { - if errors.Is(err, regerr.ErrNotFound) { + if c.isNotFoundError(err) { return nil, errdefs.ErrNotFound } @@ -277,7 +280,7 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadClo // check if the blob exists so we can bail early _, err = c.rc.BlobHead(ctx, c.ref, c.convertDescriptorToRegClient(desc)) if err != nil { - if errors.Is(err, regerr.ErrNotFound) { + if c.isNotFoundError(err) { return nil, errdefs.ErrNotFound } @@ -315,3 +318,7 @@ func (c *Client) isManifest(desc ociv1.Descriptor) bool { return false } + +func (c *Client) isNotFoundError(err error) bool { + return errors.Is(err, regerr.ErrNotFound) || strings.Contains(err.Error(), regerr.ErrNotFound.Error()) +} From 8287d4539ef0433dd77a1a5cb8dcf8bb1488145e Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:42:43 +0100 Subject: [PATCH 09/17] trying out the entire flow with ORAS instead Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> --- .../repositories/ocireg/repository.go | 48 +++-- .../extensions/repositories/ocireg/utils.go | 2 + api/tech/oras/client.go | 204 ++++++++++++++++++ api/tech/oras/delayed_reader.go | 56 +++++ go.mod | 1 + go.sum | 2 + 6 files changed, 291 insertions(+), 22 deletions(-) create mode 100644 api/tech/oras/delayed_reader.go diff --git a/api/oci/extensions/repositories/ocireg/repository.go b/api/oci/extensions/repositories/ocireg/repository.go index 3183f085d..9b9faf680 100644 --- a/api/oci/extensions/repositories/ocireg/repository.go +++ b/api/oci/extensions/repositories/ocireg/repository.go @@ -8,7 +8,9 @@ import ( "github.com/containerd/errdefs" "github.com/mandelsoft/goutils/errors" "github.com/mandelsoft/logging" - regconfig "github.com/regclient/regclient/config" + "ocm.software/ocm/api/tech/oras" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" "ocm.software/ocm/api/credentials" "ocm.software/ocm/api/oci/artdesc" @@ -120,30 +122,27 @@ func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { if creds == nil { logger.Trace("no credentials") } - - var ( - password string - username string - ) - + + authCreds := auth.Credential{} if creds != nil { - password = creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) - if password == "" { - password = creds.GetProperty(credentials.ATTR_PASSWORD) + pass := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) + if pass == "" { + pass = creds.GetProperty(credentials.ATTR_PASSWORD) } - username = creds.GetProperty(credentials.ATTR_USERNAME) + authCreds.Username = creds.GetProperty(credentials.ATTR_USERNAME) + authCreds.Password = pass } - opts := regclient.ClientOptions{ - Host: []regconfig.Host{ - { - Name: "ghcr.io", //TODO: Need to figure out how to set the host. - User: username, - Pass: password, - }, - }, - Version: comp, - } + //opts := regclient.ClientOptions{ + // Host: []regconfig.Host{ + // { + // Name: "ghcr.io", //TODO: Need to figure out how to set the host. + // User: username, + // Pass: password, + // }, + // }, + // Version: comp, + //} //opts := docker.ResolverOptions{ // Hosts: docker.ConvertHosts(config.ConfigureHosts(context.Background(), config.HostOptions{ // UpdateClient: func(client *http.Client) error { @@ -193,8 +192,13 @@ func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { // }(), // })), //} + client := &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential(r.info.HostPort(), authCreds), + } - return regclient.New(opts), nil + return oras.New(oras.ClientOptions{Client: client}), nil } func (r *RepositoryImpl) GetRef(comp, vers string) string { diff --git a/api/oci/extensions/repositories/ocireg/utils.go b/api/oci/extensions/repositories/ocireg/utils.go index 8c58c5237..afed64eb2 100644 --- a/api/oci/extensions/repositories/ocireg/utils.go +++ b/api/oci/extensions/repositories/ocireg/utils.go @@ -92,6 +92,8 @@ func pushData(ctx context.Context, p regclient.Pusher, desc artdesc.Descriptor, desc.Size = -1 } + fmt.Println("pushData: ", desc, key) + logging.Logger().Debug("*** push blob", "mediatype", desc.MediaType, "digest", desc.Digest, "key", key) req, err := p.Push(ctx, desc, data) if err != nil { diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index d2c233c24..0d1d4ce68 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -1 +1,205 @@ package oras + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/opencontainers/go-digest" + ociv1 "github.com/opencontainers/image-spec/specs-go/v1" + regref "github.com/regclient/regclient/types/ref" + "ocm.software/ocm/api/tech/regclient" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" +) + +type ClientOptions struct { + Client *auth.Client + PlainHTTP bool +} + +type Client struct { + Client *auth.Client + //Repository *remote.Repository + PlainHTTP bool + Ref string +} + +type pushRequest struct { + ref regref.Ref +} + +func (p *pushRequest) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { + return nil +} + +func (p *pushRequest) Status() (content.Status, error) { + return content.Status{}, nil +} + +var _ regclient.PushRequest = &pushRequest{} + +var ( + _ regclient.Resolver = &Client{} + _ regclient.Fetcher = &Client{} + _ regclient.Pusher = &Client{} + _ regclient.Lister = &Client{} +) + +func New(opts ClientOptions) *Client { + return &Client{Client: opts.Client, PlainHTTP: opts.PlainHTTP} +} + +// Close must be called at the end of the operation. +func (c *Client) Close(ctx context.Context, ref regref.Ref) error { + return nil +} + +func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { + src, err := c.resolveRef(ref) + if err != nil { + return "", ociv1.Descriptor{}, err + } + + // We try to first resolve a manifest. + desc, err := src.Resolve(ctx, ref) + if err != nil { + if strings.Contains(err.Error(), "not found") { + // Then we use the blob store to resolve. + desc, err := src.Blobs().Resolve(ctx, ref) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return "", ociv1.Descriptor{}, errdefs.ErrNotFound + } + + return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob: %w", err) + } + + return ref, desc, nil + } + + return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve manifest %q: %w", ref, err) + } + + return "", desc, nil +} + +func (c *Client) Fetcher(ctx context.Context, ref string) (regclient.Fetcher, error) { + c.Ref = ref + return c, nil +} + +func (c *Client) Pusher(ctx context.Context, ref string) (regclient.Pusher, error) { + c.Ref = ref + return c, nil +} + +func (c *Client) Lister(ctx context.Context, ref string) (regclient.Lister, error) { + c.Ref = ref + return c, nil +} + +func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src regclient.Source) (regclient.PushRequest, error) { + reader, err := src.Reader() + if err != nil { + return nil, err + } + + repository, err := c.resolveRef(c.Ref) + if err != nil { + return nil, err + } + + if split := strings.Split(c.Ref, ":"); len(split) == 2 { + // Once we get a reference that contains a tag, we need to re-push that + // layer with the reference included. PushReference pushes a blob or a + // manifest. + if err := repository.PushReference(ctx, d, reader, c.Ref); err != nil { + return nil, fmt.Errorf("failed to push tag: %w", err) + } + + return &pushRequest{}, nil + } + + // We have a digest, so we push use plain push for the digest. + // Push here decides if it's a Manifest or a Blob. + if err := repository.Push(ctx, d, reader); err != nil { + return nil, fmt.Errorf("failed to push: %w, %s", err, c.Ref) + } + + //ref, err := registry.ParseReference(c.Ref) + //if err != nil { + // return nil, err + //} + //if ref. + //if err := repository.Tag(ctx, d, c.Ref); err != nil { + // return nil, fmt.Errorf("failed to push tag: %w", err) + //} + + return &pushRequest{}, nil +} + +func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadCloser, error) { + src, err := c.resolveRef(c.Ref) + if err != nil { + return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) + } + + // manifest is not set in the descriptor + // src.Resolve is a manifest().resolve + rdesc, err := src.Resolve(ctx, desc.Digest.String()) + if err != nil { + if strings.Contains(err.Error(), "not found") { + rdesc, err = src.Blobs().Resolve(ctx, desc.Digest.String()) + if err != nil { + return nil, fmt.Errorf("failed to resolve fetch blob %q: %w", desc.Digest.String(), err) + } + delayer := func() (io.ReadCloser, error) { + return src.Blobs().Fetch(ctx, rdesc) + } + + return newDelayedReader(delayer) + } + + return nil, fmt.Errorf("failed to resolve fetch manifest %q: %w", desc.Digest.String(), err) + } + + fetch, err := src.Fetch(ctx, rdesc) + if err != nil { + return nil, fmt.Errorf("failed to fetch manifest: %w", err) + } + + return fetch, err +} + +func (c *Client) List(ctx context.Context) ([]string, error) { + var result []string + src, err := c.resolveRef(c.Ref) + if err != nil { + return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) + } + + if err := src.Tags(ctx, "", func(tags []string) error { + result = append(result, tags...) + return nil + }); err != nil { + return nil, fmt.Errorf("failed to list tags: %w", err) + } + + return result, nil +} + +func (c *Client) resolveRef(ref string) (*remote.Repository, error) { + src, err := remote.NewRepository(ref) + if err != nil { + return nil, fmt.Errorf("failed to create new repository: %w", err) + } + + src.Client = c.Client // set up authenticated client. + src.PlainHTTP = c.PlainHTTP + + return src, nil +} diff --git a/api/tech/oras/delayed_reader.go b/api/tech/oras/delayed_reader.go new file mode 100644 index 000000000..b73055cc0 --- /dev/null +++ b/api/tech/oras/delayed_reader.go @@ -0,0 +1,56 @@ +package oras + +import "io" + +// delayedReader sets up a reader that only fetches a blob +// upon explicit reading request, otherwise, it stores the +// way of getting the reader. +type delayedReader struct { + open func() (io.ReadCloser, error) + rc io.ReadCloser + closed bool +} + +func newDelayedReader(open func() (io.ReadCloser, error)) (*delayedReader, error) { + return &delayedReader{ + open: open, + }, nil +} + +func (d *delayedReader) Read(p []byte) (n int, err error) { + if d.closed { + return 0, io.EOF + } + + reader, err := d.reader() + if err != nil { + return 0, err + + } + + return reader.Read(p) +} + +func (d *delayedReader) reader() (io.ReadCloser, error) { + if d.rc != nil { + return d.rc, nil + } + + rc, err := d.open() + if err != nil { + return nil, err + } + + d.rc = rc + return rc, nil +} + +func (d *delayedReader) Close() error { + if d.closed { + return nil + } + + // we close regardless of an error + d.closed = true + return d.rc.Close() +} diff --git a/go.mod b/go.mod index 69cbffe6c..7174b0653 100644 --- a/go.mod +++ b/go.mod @@ -84,6 +84,7 @@ require ( k8s.io/apimachinery v0.31.3 k8s.io/cli-runtime v0.31.3 k8s.io/client-go v0.31.3 + oras.land/oras-go/v2 v2.5.0 sigs.k8s.io/controller-runtime v0.19.3 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 8eecd224a..386721667 100644 --- a/go.sum +++ b/go.sum @@ -1364,6 +1364,8 @@ k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 h1:jGnCPejIetjiy2gqaJ5V0NLwTpF4w k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.6 h1:z8cmxQXBU8yZ4mkytWqXfo6tZcamPwjsuxYU81xJ8Lk= oras.land/oras-go v1.2.6/go.mod h1:OVPc1PegSEe/K8YiLfosrlqlqTN9PUyFvOw5Y9gwrT8= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= From 7358582b99ad9182026fe747ef081f85d050b2ae Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:45:19 +0100 Subject: [PATCH 10/17] removed fmt.Println --- api/oci/extensions/repositories/ocireg/utils.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/oci/extensions/repositories/ocireg/utils.go b/api/oci/extensions/repositories/ocireg/utils.go index afed64eb2..8c58c5237 100644 --- a/api/oci/extensions/repositories/ocireg/utils.go +++ b/api/oci/extensions/repositories/ocireg/utils.go @@ -92,8 +92,6 @@ func pushData(ctx context.Context, p regclient.Pusher, desc artdesc.Descriptor, desc.Size = -1 } - fmt.Println("pushData: ", desc, key) - logging.Logger().Debug("*** push blob", "mediatype", desc.MediaType, "digest", desc.Digest, "key", key) req, err := p.Push(ctx, desc, data) if err != nil { From f7ce4578c63039e2ce676c03b4c813d9ae6e2d40 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:00:34 +0100 Subject: [PATCH 11/17] fixin some lintin --- .../repositories/ocireg/repository.go | 63 +------------------ api/tech/oras/client.go | 25 ++------ api/tech/oras/delayed_reader.go | 1 - api/tech/regclient/client.go | 2 +- api/tech/regclient/delayed_reader.go | 1 - 5 files changed, 7 insertions(+), 85 deletions(-) diff --git a/api/oci/extensions/repositories/ocireg/repository.go b/api/oci/extensions/repositories/ocireg/repository.go index 9b9faf680..62eab2280 100644 --- a/api/oci/extensions/repositories/ocireg/repository.go +++ b/api/oci/extensions/repositories/ocireg/repository.go @@ -8,7 +8,6 @@ import ( "github.com/containerd/errdefs" "github.com/mandelsoft/goutils/errors" "github.com/mandelsoft/logging" - "ocm.software/ocm/api/tech/oras" "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2/registry/remote/retry" @@ -16,6 +15,7 @@ import ( "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/tech/oci/identity" + "ocm.software/ocm/api/tech/oras" "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/utils" ocmlog "ocm.software/ocm/api/utils/logging" @@ -122,7 +122,7 @@ func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { if creds == nil { logger.Trace("no credentials") } - + authCreds := auth.Credential{} if creds != nil { pass := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) @@ -133,65 +133,6 @@ func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { authCreds.Password = pass } - //opts := regclient.ClientOptions{ - // Host: []regconfig.Host{ - // { - // Name: "ghcr.io", //TODO: Need to figure out how to set the host. - // User: username, - // Pass: password, - // }, - // }, - // Version: comp, - //} - //opts := docker.ResolverOptions{ - // Hosts: docker.ConvertHosts(config.ConfigureHosts(context.Background(), config.HostOptions{ - // UpdateClient: func(client *http.Client) error { - // // copy from http.DefaultTransport with a roundtripper injection - // client.Transport = ocmlog.NewRoundTripper(client.Transport, logger) - // return nil - // }, - // Credentials: func(host string) (string, string, error) { - // if creds != nil { - // p := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN) - // if p == "" { - // p = creds.GetProperty(credentials.ATTR_PASSWORD) - // } - // pw := "" - // if p != "" { - // pw = "***" - // } - // logger.Trace("query credentials", ocmlog.ATTR_USER, creds.GetProperty(credentials.ATTR_USERNAME), "pass", pw) - // return creds.GetProperty(credentials.ATTR_USERNAME), p, nil - // } - // logger.Trace("no credentials") - // return "", "", nil - // }, - // DefaultScheme: r.info.Scheme, - // //nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version. - // DefaultTLS: func() *tls.Config { - // if r.info.Scheme == "http" { - // return nil - // } - // return &tls.Config{ - // // MinVersion: tls.VersionTLS13, - // RootCAs: func() *x509.CertPool { - // var rootCAs *x509.CertPool - // if creds != nil { - // c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY) - // if c != "" { - // rootCAs = x509.NewCertPool() - // rootCAs.AppendCertsFromPEM([]byte(c)) - // } - // } - // if rootCAs == nil { - // rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true) - // } - // return rootCAs - // }(), - // } - // }(), - // })), - //} client := &auth.Client{ Client: retry.DefaultClient, Cache: auth.NewCache(), diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index 0d1d4ce68..a62328aca 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -10,10 +10,10 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/opencontainers/go-digest" ociv1 "github.com/opencontainers/image-spec/specs-go/v1" - regref "github.com/regclient/regclient/types/ref" - "ocm.software/ocm/api/tech/regclient" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" + + "ocm.software/ocm/api/tech/regclient" ) type ClientOptions struct { @@ -22,15 +22,12 @@ type ClientOptions struct { } type Client struct { - Client *auth.Client - //Repository *remote.Repository + Client *auth.Client PlainHTTP bool Ref string } -type pushRequest struct { - ref regref.Ref -} +type pushRequest struct{} func (p *pushRequest) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { return nil @@ -53,11 +50,6 @@ func New(opts ClientOptions) *Client { return &Client{Client: opts.Client, PlainHTTP: opts.PlainHTTP} } -// Close must be called at the end of the operation. -func (c *Client) Close(ctx context.Context, ref regref.Ref) error { - return nil -} - func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { src, err := c.resolveRef(ref) if err != nil { @@ -130,15 +122,6 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src regclient.Sou return nil, fmt.Errorf("failed to push: %w, %s", err, c.Ref) } - //ref, err := registry.ParseReference(c.Ref) - //if err != nil { - // return nil, err - //} - //if ref. - //if err := repository.Tag(ctx, d, c.Ref); err != nil { - // return nil, fmt.Errorf("failed to push tag: %w", err) - //} - return &pushRequest{}, nil } diff --git a/api/tech/oras/delayed_reader.go b/api/tech/oras/delayed_reader.go index b73055cc0..9918b78c5 100644 --- a/api/tech/oras/delayed_reader.go +++ b/api/tech/oras/delayed_reader.go @@ -25,7 +25,6 @@ func (d *delayedReader) Read(p []byte) (n int, err error) { reader, err := d.reader() if err != nil { return 0, err - } return reader.Read(p) diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go index c6fc2d8be..cd51397b6 100644 --- a/api/tech/regclient/client.go +++ b/api/tech/regclient/client.go @@ -148,7 +148,7 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ Digest: digest.Digest(r.Digest), }) - defer blob.Close() // we can safely close it as this is not when we read it. + defer blob.Close() //nolint:staticcheck // we can safely close it as this is not when we read it. if err != nil { if c.isNotFoundError(err) { diff --git a/api/tech/regclient/delayed_reader.go b/api/tech/regclient/delayed_reader.go index 19fb9f220..ac7f3d5fa 100644 --- a/api/tech/regclient/delayed_reader.go +++ b/api/tech/regclient/delayed_reader.go @@ -25,7 +25,6 @@ func (d *delayedReader) Read(p []byte) (n int, err error) { reader, err := d.reader() if err != nil { return 0, err - } return reader.Read(p) From c9e283010c4e4e305f4567681ad1e256722196d2 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:18:22 +0100 Subject: [PATCH 12/17] renamed the createRepository function and fixed digest resolve --- api/tech/oras/client.go | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index a62328aca..34972532a 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -51,26 +51,20 @@ func New(opts ClientOptions) *Client { } func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { - src, err := c.resolveRef(ref) + src, err := c.createRepository(ref) if err != nil { return "", ociv1.Descriptor{}, err } // We try to first resolve a manifest. + // _Note_: If there is an error like not found, but we know that the digest exists + // we can add src.Blobs().Resolve in here. If we do that, note that + // for Blobs().Resolve `not found` is actually `invalid checksum digest format`. + // Meaning it will throw that error instead of not found. desc, err := src.Resolve(ctx, ref) if err != nil { if strings.Contains(err.Error(), "not found") { - // Then we use the blob store to resolve. - desc, err := src.Blobs().Resolve(ctx, ref) - if err != nil { - if strings.Contains(err.Error(), "not found") { - return "", ociv1.Descriptor{}, errdefs.ErrNotFound - } - - return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob: %w", err) - } - - return ref, desc, nil + return "", ociv1.Descriptor{}, errdefs.ErrNotFound } return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve manifest %q: %w", ref, err) @@ -100,15 +94,16 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src regclient.Sou return nil, err } - repository, err := c.resolveRef(c.Ref) + repository, err := c.createRepository(c.Ref) if err != nil { return nil, err } if split := strings.Split(c.Ref, ":"); len(split) == 2 { // Once we get a reference that contains a tag, we need to re-push that - // layer with the reference included. PushReference pushes a blob or a - // manifest. + // layer with the reference included. PushReference then will tag + // that layer resulting in the created tag pointing to the right + // blob data. if err := repository.PushReference(ctx, d, reader, c.Ref); err != nil { return nil, fmt.Errorf("failed to push tag: %w", err) } @@ -126,14 +121,17 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src regclient.Sou } func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadCloser, error) { - src, err := c.resolveRef(c.Ref) + src, err := c.createRepository(c.Ref) if err != nil { return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) } + // oras requires a Resolve to happen before a fetch because + // -1 is an invalid size. // manifest is not set in the descriptor - // src.Resolve is a manifest().resolve - rdesc, err := src.Resolve(ctx, desc.Digest.String()) + // We explicitly call resolve on manifest first because it might be + // that the mediatype is not set at this point. + rdesc, err := src.Manifests().Resolve(ctx, desc.Digest.String()) if err != nil { if strings.Contains(err.Error(), "not found") { rdesc, err = src.Blobs().Resolve(ctx, desc.Digest.String()) @@ -150,6 +148,7 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadClose return nil, fmt.Errorf("failed to resolve fetch manifest %q: %w", desc.Digest.String(), err) } + // lastly, try a manifest fetch. fetch, err := src.Fetch(ctx, rdesc) if err != nil { return nil, fmt.Errorf("failed to fetch manifest: %w", err) @@ -160,7 +159,7 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadClose func (c *Client) List(ctx context.Context) ([]string, error) { var result []string - src, err := c.resolveRef(c.Ref) + src, err := c.createRepository(c.Ref) if err != nil { return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) } @@ -175,7 +174,7 @@ func (c *Client) List(ctx context.Context) ([]string, error) { return result, nil } -func (c *Client) resolveRef(ref string) (*remote.Repository, error) { +func (c *Client) createRepository(ref string) (*remote.Repository, error) { src, err := remote.NewRepository(ref) if err != nil { return nil, fmt.Errorf("failed to create new repository: %w", err) From 78d2bc8b31740b429f52f00f97899afda32714c8 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:47:02 +0100 Subject: [PATCH 13/17] remove regctl and create the right oras client Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> --- .../extensions/repositories/ocireg/blobs.go | 16 +- .../repositories/ocireg/namespace.go | 12 +- .../repositories/ocireg/repository.go | 44 ++- .../extensions/repositories/ocireg/utils.go | 6 +- api/tech/docker/lister.go | 4 +- api/tech/docker/pusher.go | 9 +- api/tech/docker/resolver.go | 11 +- api/tech/oras/client.go | 20 +- api/tech/{regclient => oras}/interface.go | 2 +- api/tech/regclient/client.go | 324 ------------------ api/tech/regclient/client_test.go | 16 - api/tech/regclient/delayed_reader.go | 55 --- go.mod | 2 - go.sum | 4 - 14 files changed, 77 insertions(+), 448 deletions(-) rename api/tech/{regclient => oras}/interface.go (99%) delete mode 100644 api/tech/regclient/client.go delete mode 100644 api/tech/regclient/client_test.go delete mode 100644 api/tech/regclient/delayed_reader.go diff --git a/api/oci/extensions/repositories/ocireg/blobs.go b/api/oci/extensions/repositories/ocireg/blobs.go index f92deba0a..d8491c266 100644 --- a/api/oci/extensions/repositories/ocireg/blobs.go +++ b/api/oci/extensions/repositories/ocireg/blobs.go @@ -10,7 +10,7 @@ import ( "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/extensions/attrs/cacheattr" - "ocm.software/ocm/api/tech/regclient" + "ocm.software/ocm/api/tech/oras" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" ) @@ -23,20 +23,20 @@ type BlobContainer interface { type blobContainer struct { accessio.StaticAllocatable - fetcher regclient.Fetcher - pusher regclient.Pusher + fetcher oras.Fetcher + pusher oras.Pusher mime string } type BlobContainers struct { lock sync.Mutex cache accessio.BlobCache - fetcher regclient.Fetcher - pusher regclient.Pusher + fetcher oras.Fetcher + pusher oras.Pusher mimes map[string]BlobContainer } -func NewBlobContainers(ctx cpi.Context, fetcher remotes.Fetcher, pusher regclient.Pusher) *BlobContainers { +func NewBlobContainers(ctx cpi.Context, fetcher remotes.Fetcher, pusher oras.Pusher) *BlobContainers { return &BlobContainers{ cache: cacheattr.Get(ctx), fetcher: fetcher, @@ -73,7 +73,7 @@ func (c *BlobContainers) Release() error { return list.Result() } -func newBlobContainer(mime string, fetcher regclient.Fetcher, pusher regclient.Pusher) *blobContainer { +func newBlobContainer(mime string, fetcher oras.Fetcher, pusher oras.Pusher) *blobContainer { return &blobContainer{ mime: mime, fetcher: fetcher, @@ -81,7 +81,7 @@ func newBlobContainer(mime string, fetcher regclient.Fetcher, pusher regclient.P } } -func NewBlobContainer(cache accessio.BlobCache, mime string, fetcher regclient.Fetcher, pusher regclient.Pusher) (BlobContainer, error) { +func NewBlobContainer(cache accessio.BlobCache, mime string, fetcher oras.Fetcher, pusher oras.Pusher) (BlobContainer, error) { c := newBlobContainer(mime, fetcher, pusher) if cache == nil { diff --git a/api/oci/extensions/repositories/ocireg/namespace.go b/api/oci/extensions/repositories/ocireg/namespace.go index cae5c05c3..dd3583d7b 100644 --- a/api/oci/extensions/repositories/ocireg/namespace.go +++ b/api/oci/extensions/repositories/ocireg/namespace.go @@ -12,7 +12,7 @@ import ( "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/oci/cpi/support" "ocm.software/ocm/api/oci/extensions/actions/oci-repository-prepare" - "ocm.software/ocm/api/tech/regclient" + "ocm.software/ocm/api/tech/oras" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" "ocm.software/ocm/api/utils/logging" @@ -22,10 +22,10 @@ import ( type NamespaceContainer struct { impl support.NamespaceAccessImpl repo *RepositoryImpl - resolver regclient.Resolver - lister regclient.Lister - fetcher regclient.Fetcher - pusher regclient.Pusher + resolver oras.Resolver + lister oras.Lister + fetcher oras.Fetcher + pusher oras.Pusher blobs *BlobContainers checked bool } @@ -69,7 +69,7 @@ func (n *NamespaceContainer) SetImplementation(impl support.NamespaceAccessImpl) n.impl = impl } -func (n *NamespaceContainer) getPusher(vers string) (regclient.Pusher, error) { +func (n *NamespaceContainer) getPusher(vers string) (oras.Pusher, error) { err := n.assureCreated() if err != nil { return nil, err diff --git a/api/oci/extensions/repositories/ocireg/repository.go b/api/oci/extensions/repositories/ocireg/repository.go index 62eab2280..391831533 100644 --- a/api/oci/extensions/repositories/ocireg/repository.go +++ b/api/oci/extensions/repositories/ocireg/repository.go @@ -2,6 +2,9 @@ package ocireg import ( "context" + "crypto/tls" + "crypto/x509" + "net/http" "path" "strings" @@ -12,11 +15,11 @@ import ( "oras.land/oras-go/v2/registry/remote/retry" "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/datacontext/attrs/rootcertsattr" "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" "ocm.software/ocm/api/tech/oci/identity" "ocm.software/ocm/api/tech/oras" - "ocm.software/ocm/api/tech/regclient" "ocm.software/ocm/api/utils" ocmlog "ocm.software/ocm/api/utils/logging" "ocm.software/ocm/api/utils/refmgmt" @@ -111,7 +114,7 @@ func (r *RepositoryImpl) getCreds(comp string) (credentials.Credentials, error) return identity.GetCredentials(r.GetContext(), r.info.Locator, comp) } -func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { +func (r *RepositoryImpl) getResolver(comp string) (oras.Resolver, error) { creds, err := r.getCreds(comp) if err != nil { if !errors.IsErrUnknownKind(err, credentials.KIND_CONSUMER) { @@ -133,13 +136,44 @@ func (r *RepositoryImpl) getResolver(comp string) (regclient.Resolver, error) { authCreds.Password = pass } - client := &auth.Client{ - Client: retry.DefaultClient, + client := retry.DefaultClient + if r.info.Scheme == "https" { + // set up TLS + //nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version. + conf := &tls.Config{ + // MinVersion: tls.VersionTLS13, + RootCAs: func() *x509.CertPool { + var rootCAs *x509.CertPool + if creds != nil { + c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY) + if c != "" { + rootCAs = x509.NewCertPool() + rootCAs.AppendCertsFromPEM([]byte(c)) + } + } + if rootCAs == nil { + rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true) + } + return rootCAs + }(), + } + client = &http.Client{ + Transport: retry.NewTransport(&http.Transport{ + TLSClientConfig: conf, + }), + } + } + + authClient := &auth.Client{ + Client: client, Cache: auth.NewCache(), Credential: auth.StaticCredential(r.info.HostPort(), authCreds), } - return oras.New(oras.ClientOptions{Client: client}), nil + return oras.New(oras.ClientOptions{ + Client: authClient, + PlainHTTP: r.info.Scheme == "http", + }), nil } func (r *RepositoryImpl) GetRef(comp, vers string) string { diff --git a/api/oci/extensions/repositories/ocireg/utils.go b/api/oci/extensions/repositories/ocireg/utils.go index 8c58c5237..761cb045c 100644 --- a/api/oci/extensions/repositories/ocireg/utils.go +++ b/api/oci/extensions/repositories/ocireg/utils.go @@ -14,7 +14,7 @@ import ( "ocm.software/ocm/api/oci/artdesc" "ocm.software/ocm/api/oci/cpi" - "ocm.software/ocm/api/tech/regclient" + "ocm.software/ocm/api/tech/oras" "ocm.software/ocm/api/utils/accessio" "ocm.software/ocm/api/utils/blobaccess/blobaccess" "ocm.software/ocm/api/utils/logging" @@ -81,12 +81,12 @@ func readAll(reader io.ReadCloser, err error) ([]byte, error) { return data, nil } -func push(ctx context.Context, p regclient.Pusher, blob blobaccess.BlobAccess) error { +func push(ctx context.Context, p oras.Pusher, blob blobaccess.BlobAccess) error { desc := *artdesc.DefaultBlobDescriptor(blob) return pushData(ctx, p, desc, blob) } -func pushData(ctx context.Context, p regclient.Pusher, desc artdesc.Descriptor, data blobaccess.DataAccess) error { +func pushData(ctx context.Context, p oras.Pusher, desc artdesc.Descriptor, data blobaccess.DataAccess) error { key := remotes.MakeRefKey(ctx, desc) if desc.Size == 0 { desc.Size = -1 diff --git a/api/tech/docker/lister.go b/api/tech/docker/lister.go index 95b7a5117..aa492f3a7 100644 --- a/api/tech/docker/lister.go +++ b/api/tech/docker/lister.go @@ -11,7 +11,7 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" "github.com/pkg/errors" - "ocm.software/ocm/api/tech/regclient" + "ocm.software/ocm/api/tech/oras" ) var ErrObjectNotRequired = errors.New("object not required") @@ -25,7 +25,7 @@ type dockerLister struct { dockerBase *dockerBase } -func (r *dockerResolver) Lister(ctx context.Context, ref string) (regclient.Lister, error) { +func (r *dockerResolver) Lister(ctx context.Context, ref string) (oras.Lister, error) { base, err := r.resolveDockerBase(ref) if err != nil { return nil, err diff --git a/api/tech/docker/pusher.go b/api/tech/docker/pusher.go index 53b2a785f..0ab86dd4d 100644 --- a/api/tech/docker/pusher.go +++ b/api/tech/docker/pusher.go @@ -19,9 +19,8 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "ocm.software/ocm/api/tech/regclient" - remoteserrors "ocm.software/ocm/api/tech/docker/errors" + "ocm.software/ocm/api/tech/oras" "ocm.software/ocm/api/utils/accessio" ) @@ -39,11 +38,11 @@ type dockerPusher struct { tracker StatusTracker } -func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor, src regclient.Source) (regclient.PushRequest, error) { +func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor, src oras.Source) (oras.PushRequest, error) { return p.push(ctx, desc, src, remotes.MakeRefKey(ctx, desc), false) } -func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, src regclient.Source, ref string, unavailableOnFail bool) (regclient.PushRequest, error) { +func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, src oras.Source, ref string, unavailableOnFail bool) (oras.PushRequest, error) { if l, ok := p.tracker.(StatusTrackLocker); ok { l.Lock(ref) defer l.Unlock(ref) @@ -324,7 +323,7 @@ type pushRequest struct { ref string responseC <-chan response - source regclient.Source + source oras.Source isManifest bool expected digest.Digest diff --git a/api/tech/docker/resolver.go b/api/tech/docker/resolver.go index a5b5f3161..231a78976 100644 --- a/api/tech/docker/resolver.go +++ b/api/tech/docker/resolver.go @@ -22,8 +22,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context/ctxhttp" - "ocm.software/ocm/api/tech/regclient" - + "ocm.software/ocm/api/tech/oras" "ocm.software/ocm/api/utils/accessio" ) @@ -120,7 +119,7 @@ type dockerResolver struct { } // NewResolver returns a new resolver to a Docker registry. -func NewResolver(options ResolverOptions) regclient.Resolver { +func NewResolver(options ResolverOptions) oras.Resolver { if options.Tracker == nil { options.Tracker = NewInMemoryTracker() } @@ -204,7 +203,7 @@ func (r *countingReader) Read(p []byte) (int, error) { return n, err } -var _ regclient.Resolver = &dockerResolver{} +var _ oras.Resolver = &dockerResolver{} func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) { base, err := r.resolveDockerBase(ref) @@ -384,7 +383,7 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp return "", ocispec.Descriptor{}, firstErr } -func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (regclient.Fetcher, error) { +func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (oras.Fetcher, error) { base, err := r.resolveDockerBase(ref) if err != nil { return nil, err @@ -395,7 +394,7 @@ func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (regclient.Fet }, nil } -func (r *dockerResolver) Pusher(ctx context.Context, ref string) (regclient.Pusher, error) { +func (r *dockerResolver) Pusher(ctx context.Context, ref string) (oras.Pusher, error) { base, err := r.resolveDockerBase(ref) if err != nil { return nil, err diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index 34972532a..bf9c3b89d 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -12,8 +12,6 @@ import ( ociv1 "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" - - "ocm.software/ocm/api/tech/regclient" ) type ClientOptions struct { @@ -37,13 +35,13 @@ func (p *pushRequest) Status() (content.Status, error) { return content.Status{}, nil } -var _ regclient.PushRequest = &pushRequest{} +var _ PushRequest = &pushRequest{} var ( - _ regclient.Resolver = &Client{} - _ regclient.Fetcher = &Client{} - _ regclient.Pusher = &Client{} - _ regclient.Lister = &Client{} + _ Resolver = &Client{} + _ Fetcher = &Client{} + _ Pusher = &Client{} + _ Lister = &Client{} ) func New(opts ClientOptions) *Client { @@ -73,22 +71,22 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip return "", desc, nil } -func (c *Client) Fetcher(ctx context.Context, ref string) (regclient.Fetcher, error) { +func (c *Client) Fetcher(ctx context.Context, ref string) (Fetcher, error) { c.Ref = ref return c, nil } -func (c *Client) Pusher(ctx context.Context, ref string) (regclient.Pusher, error) { +func (c *Client) Pusher(ctx context.Context, ref string) (Pusher, error) { c.Ref = ref return c, nil } -func (c *Client) Lister(ctx context.Context, ref string) (regclient.Lister, error) { +func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { c.Ref = ref return c, nil } -func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src regclient.Source) (regclient.PushRequest, error) { +func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (PushRequest, error) { reader, err := src.Reader() if err != nil { return nil, err diff --git a/api/tech/regclient/interface.go b/api/tech/oras/interface.go similarity index 99% rename from api/tech/regclient/interface.go rename to api/tech/oras/interface.go index d11f0648d..ccd9ab1d4 100644 --- a/api/tech/regclient/interface.go +++ b/api/tech/oras/interface.go @@ -1,4 +1,4 @@ -package regclient +package oras import ( "context" diff --git a/api/tech/regclient/client.go b/api/tech/regclient/client.go deleted file mode 100644 index cd51397b6..000000000 --- a/api/tech/regclient/client.go +++ /dev/null @@ -1,324 +0,0 @@ -package regclient - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "strings" - "time" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/images" - "github.com/opencontainers/go-digest" - ociv1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/regclient/regclient" - "github.com/regclient/regclient/config" - "github.com/regclient/regclient/scheme/reg" - "github.com/regclient/regclient/types/descriptor" - regerr "github.com/regclient/regclient/types/errs" - "github.com/regclient/regclient/types/manifest" - "github.com/regclient/regclient/types/platform" - regref "github.com/regclient/regclient/types/ref" -) - -type ClientOptions struct { - Host []config.Host - Version string -} - -type Client struct { - rc *regclient.RegClient - ref regref.Ref -} - -type pushRequest struct { - rc *regclient.RegClient - desc descriptor.Descriptor - ref regref.Ref -} - -// Commit and Status are actually not really used in the library. Commit is a second stage operation and Status is never called in -// the library. The Status was a thing mostly in docker being used to track chunk reads. But that's taken care of -// by regclient internally. -func (p *pushRequest) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { - return p.rc.Close(ctx, p.ref) -} - -func (p *pushRequest) Status() (content.Status, error) { - return content.Status{ - Ref: p.ref.Reference, - Total: p.desc.Size, - }, nil -} - -var _ PushRequest = &pushRequest{} - -var ( - _ Resolver = &Client{} - _ Fetcher = &Client{} - _ Pusher = &Client{} - _ Lister = &Client{} -) - -func New(opts ClientOptions) *Client { - rc := regclient.New( - regclient.WithConfigHost(opts.Host...), - regclient.WithDockerCerts(), - regclient.WithDockerCreds(), - regclient.WithUserAgent("containerd/"+opts.Version), - regclient.WithRegOpts( - // reg.WithCertDirs([]string{"."}), - reg.WithDelay(2*time.Second, 15*time.Second), - reg.WithRetryLimit(5), - reg.WithCache(5*time.Minute, 500), // built in cache!! Nice! - ), - ) - - return &Client{rc: rc} -} - -// Close must be called at the end of the operation. -func (c *Client) Close(ctx context.Context, ref regref.Ref) error { - return c.rc.Close(ctx, ref) -} - -func (c *Client) convertDescriptorToOCI(desc descriptor.Descriptor) ociv1.Descriptor { - var p *ociv1.Platform - if desc.Platform != nil { - p = &ociv1.Platform{ - Architecture: desc.Platform.Architecture, - OS: desc.Platform.OS, - OSVersion: desc.Platform.OSVersion, - OSFeatures: desc.Platform.OSFeatures, - Variant: desc.Platform.Variant, - } - } - - return ociv1.Descriptor{ - MediaType: desc.MediaType, - Size: desc.Size, - Digest: desc.Digest, - Platform: p, - URLs: desc.URLs, - Annotations: desc.Annotations, - Data: desc.Data, - ArtifactType: desc.ArtifactType, - } -} - -func (c *Client) convertDescriptorToRegClient(desc ociv1.Descriptor) descriptor.Descriptor { - var p *platform.Platform - if desc.Platform != nil { - p = &platform.Platform{ - Architecture: desc.Platform.Architecture, - OS: desc.Platform.OS, - OSVersion: desc.Platform.OSVersion, - OSFeatures: desc.Platform.OSFeatures, - Variant: desc.Platform.Variant, - } - } - - return descriptor.Descriptor{ - MediaType: desc.MediaType, - Size: desc.Size, - Digest: desc.Digest, - Platform: p, - URLs: desc.URLs, - Annotations: desc.Annotations, - Data: desc.Data, - ArtifactType: desc.ArtifactType, - } -} - -func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { - r, err := regref.New(ref) - if err != nil { - return "", ociv1.Descriptor{}, err - } - - // first, try to find the manifest - m, err := c.rc.ManifestHead(ctx, r) - if err != nil { - if c.isNotFoundError(err) { - // try to find a blob - if r.Digest != "" { - blob, err := c.rc.BlobHead(ctx, r, descriptor.Descriptor{ - Digest: digest.Digest(r.Digest), - }) - defer blob.Close() //nolint:staticcheck // we can safely close it as this is not when we read it. - - if err != nil { - if c.isNotFoundError(err) { - return "", ociv1.Descriptor{}, errdefs.ErrNotFound - } - - return "", ociv1.Descriptor{}, fmt.Errorf("failed to resolve blob head: %w", err) - } - - // update the reference that has been resolved successfully - c.ref = r - - return ref, c.convertDescriptorToOCI(blob.GetDescriptor()), nil - } - - return "", ociv1.Descriptor{}, errdefs.ErrNotFound - } - - return "", ociv1.Descriptor{}, fmt.Errorf("failed to get manifest: %w", err) - } - - // update the Ref of the client to the resolved reference. - c.ref = r - - return ref, c.convertDescriptorToOCI(m.GetDescriptor()), nil -} - -func (c *Client) Fetcher(ctx context.Context, ref string) (Fetcher, error) { - var err error - c.ref, err = regref.New(ref) - if err != nil { - return nil, err - } - - return c, nil -} - -func (c *Client) Pusher(ctx context.Context, ref string) (Pusher, error) { - var err error - c.ref, err = regref.New(ref) - if err != nil { - return nil, err - } - - return c, nil -} - -func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { - var err error - c.ref, err = regref.New(ref) - if err != nil { - return nil, err - } - - return c, nil -} - -func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (PushRequest, error) { - reader, err := src.Reader() - if err != nil { - return nil, err - } - - if c.isManifest(d) { - manifestContent, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed to read manifest: %w", err) - } - - m, err := manifest.New(manifest.WithDesc(c.convertDescriptorToRegClient(d)), manifest.WithRef(c.ref), manifest.WithRaw(manifestContent)) - if err != nil { - return nil, fmt.Errorf("failed to create a manifest: %w", err) - } - - if err := c.rc.ManifestPut(ctx, c.ref, m); err != nil { - return nil, fmt.Errorf("failed to put manifest: %w", err) - } - - // pushRequest closes the RC on `Commit`. - return &pushRequest{ - desc: c.convertDescriptorToRegClient(d), - rc: c.rc, - ref: c.ref, - }, nil - } - - desc, err := c.rc.BlobPut(ctx, c.ref, c.convertDescriptorToRegClient(d), reader) - if err != nil { - return nil, err - } - - return &pushRequest{ - desc: desc, - rc: c.rc, - ref: c.ref, - }, nil -} - -func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (_ io.ReadCloser, err error) { - defer func() { - if cerr := c.rc.Close(ctx, c.ref); cerr != nil { - err = errors.Join(err, fmt.Errorf("failed to close the client after fetch: %w", cerr)) - } - }() - - // -1 is not a thing in regclient. - if desc.Size < 0 { - desc.Size = 0 - } - - if c.isManifest(desc) { - manifestContent, err := c.rc.ManifestGet(ctx, c.ref, regclient.WithManifestDesc(c.convertDescriptorToRegClient(desc))) - if err != nil { - if c.isNotFoundError(err) { - return nil, errdefs.ErrNotFound - } - - return nil, err - } - - body, err := manifestContent.RawBody() - if err != nil { - return nil, err - } - - return io.NopCloser(bytes.NewReader(body)), nil - } - - // check if the blob exists so we can bail early - _, err = c.rc.BlobHead(ctx, c.ref, c.convertDescriptorToRegClient(desc)) - if err != nil { - if c.isNotFoundError(err) { - return nil, errdefs.ErrNotFound - } - - return nil, err - } - - delayer := func() (io.ReadCloser, error) { - return c.rc.BlobGet(ctx, c.ref, c.convertDescriptorToRegClient(desc)) - } - - return newDelayedReader(delayer) -} - -func (c *Client) List(ctx context.Context) (_ []string, err error) { - defer func() { - if cerr := c.rc.Close(ctx, c.ref); cerr != nil { - err = errors.Join(err, fmt.Errorf("failed to close the client after list: %w", cerr)) - } - }() - - tags, err := c.rc.TagList(ctx, c.ref) - if err != nil { - return nil, err - } - - return tags.Tags, nil -} - -func (c *Client) isManifest(desc ociv1.Descriptor) bool { - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, - ociv1.MediaTypeImageManifest, ociv1.MediaTypeImageIndex: - return true - } - - return false -} - -func (c *Client) isNotFoundError(err error) bool { - return errors.Is(err, regerr.ErrNotFound) || strings.Contains(err.Error(), regerr.ErrNotFound.Error()) -} diff --git a/api/tech/regclient/client_test.go b/api/tech/regclient/client_test.go deleted file mode 100644 index b5045d702..000000000 --- a/api/tech/regclient/client_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package regclient - -import ( - "testing" - - "github.com/regclient/regclient/config" - "github.com/stretchr/testify/require" -) - -func TestNew(t *testing.T) { - host := config.HostNew() - n := New(ClientOptions{Host: []config.Host{ - *host, - }}) - require.NotNil(t, n) -} diff --git a/api/tech/regclient/delayed_reader.go b/api/tech/regclient/delayed_reader.go deleted file mode 100644 index ac7f3d5fa..000000000 --- a/api/tech/regclient/delayed_reader.go +++ /dev/null @@ -1,55 +0,0 @@ -package regclient - -import "io" - -// delayedReader sets up a reader that only fetches a blob -// upon explicit reading request, otherwise, it stores the -// way of getting the reader. -type delayedReader struct { - open func() (io.ReadCloser, error) - rc io.ReadCloser - closed bool -} - -func newDelayedReader(open func() (io.ReadCloser, error)) (io.ReadCloser, error) { - return &delayedReader{ - open: open, - }, nil -} - -func (d *delayedReader) Read(p []byte) (n int, err error) { - if d.closed { - return 0, io.EOF - } - - reader, err := d.reader() - if err != nil { - return 0, err - } - - return reader.Read(p) -} - -func (d *delayedReader) reader() (io.ReadCloser, error) { - if d.rc != nil { - return d.rc, nil - } - - rc, err := d.open() - if err != nil { - return nil, err - } - - d.rc = rc - return rc, nil -} - -func (d *delayedReader) Close() error { - if d.closed { - return nil - } - - // we close regardless of an error - d.closed = true - return d.rc.Close() -} diff --git a/go.mod b/go.mod index ac7d4f164..e2c53c704 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,6 @@ require ( github.com/opencontainers/image-spec v1.1.0 github.com/pkg/errors v0.9.1 github.com/redis/go-redis/v9 v9.7.0 - github.com/regclient/regclient v0.7.2 github.com/rogpeppe/go-internal v1.13.1 github.com/sigstore/cosign/v2 v2.4.1 github.com/sigstore/rekor v1.3.7 @@ -176,7 +175,6 @@ require ( github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/orderedmap v1.7.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect diff --git a/go.sum b/go.sum index d0c118cd9..cfe049224 100644 --- a/go.sum +++ b/go.sum @@ -783,8 +783,6 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olareg/olareg v0.1.1 h1:Ui7q93zjcoF+U9U71sgqgZWByDoZOpqHitUXEu2xV+g= -github.com/olareg/olareg v0.1.1/go.mod h1:w8NP4SWrHHtxsFaUiv1lnCnYPm4sN1seCd2h7FK/dc0= github.com/oleiade/reflections v1.1.0 h1:D+I/UsXQB4esMathlt0kkZRJZdUDmhv5zGi/HOwYTWo= github.com/oleiade/reflections v1.1.0/go.mod h1:mCxx0QseeVCHs5Um5HhJeCKVC7AwS8kO67tky4rdisA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -874,8 +872,6 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.5.3 h1:kuvuJL/+MZIEdvtb/kTBRiRgY github.com/redis/go-redis/extra/redisotel/v9 v9.5.3/go.mod h1:7f/FMrf5RRRVHXgfk7CzSVzXHiWeuOQUu2bsVqWoa+g= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= -github.com/regclient/regclient v0.7.2 h1:vcldDAwBMLtighYVMeb6qNt5+0hKg3AN2IkCc0JIJNM= -github.com/regclient/regclient v0.7.2/go.mod h1:QlA7W9/pvmbblOXM4d49JgfuOTwVXcUMKt3bFuOSVIQ= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= From 024a606e51e75f9e9622610a04701d61e730ecef Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:51:04 +0100 Subject: [PATCH 14/17] simplified the interface and removed the unused status and commit --- .../extensions/repositories/ocireg/utils.go | 5 ++- api/tech/oras/client.go | 34 ++++++------------- api/tech/oras/interface.go | 17 +--------- 3 files changed, 14 insertions(+), 42 deletions(-) diff --git a/api/oci/extensions/repositories/ocireg/utils.go b/api/oci/extensions/repositories/ocireg/utils.go index 761cb045c..51f5994d1 100644 --- a/api/oci/extensions/repositories/ocireg/utils.go +++ b/api/oci/extensions/repositories/ocireg/utils.go @@ -93,8 +93,7 @@ func pushData(ctx context.Context, p oras.Pusher, desc artdesc.Descriptor, data } logging.Logger().Debug("*** push blob", "mediatype", desc.MediaType, "digest", desc.Digest, "key", key) - req, err := p.Push(ctx, desc, data) - if err != nil { + if err := p.Push(ctx, desc, data); err != nil { if errdefs.IsAlreadyExists(err) { logging.Logger().Debug("blob already exists", "mediatype", desc.MediaType, "digest", desc.Digest) @@ -104,7 +103,7 @@ func pushData(ctx context.Context, p oras.Pusher, desc artdesc.Descriptor, data return fmt.Errorf("failed to push: %w", err) } - return req.Commit(ctx, desc.Size, desc.Digest) + return nil } var dummyContext = nologger() diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index bf9c3b89d..6ce26019c 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -2,14 +2,14 @@ package oras import ( "context" + "errors" "fmt" "io" "strings" - "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" - "github.com/opencontainers/go-digest" ociv1 "github.com/opencontainers/image-spec/specs-go/v1" + oraserr "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" ) @@ -25,18 +25,6 @@ type Client struct { Ref string } -type pushRequest struct{} - -func (p *pushRequest) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { - return nil -} - -func (p *pushRequest) Status() (content.Status, error) { - return content.Status{}, nil -} - -var _ PushRequest = &pushRequest{} - var ( _ Resolver = &Client{} _ Fetcher = &Client{} @@ -61,7 +49,7 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip // Meaning it will throw that error instead of not found. desc, err := src.Resolve(ctx, ref) if err != nil { - if strings.Contains(err.Error(), "not found") { + if errors.Is(err, oraserr.ErrNotFound) { return "", ociv1.Descriptor{}, errdefs.ErrNotFound } @@ -86,15 +74,15 @@ func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { return c, nil } -func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (PushRequest, error) { +func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error { reader, err := src.Reader() if err != nil { - return nil, err + return err } repository, err := c.createRepository(c.Ref) if err != nil { - return nil, err + return err } if split := strings.Split(c.Ref, ":"); len(split) == 2 { @@ -103,19 +91,19 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) (Push // that layer resulting in the created tag pointing to the right // blob data. if err := repository.PushReference(ctx, d, reader, c.Ref); err != nil { - return nil, fmt.Errorf("failed to push tag: %w", err) + return fmt.Errorf("failed to push tag: %w", err) } - return &pushRequest{}, nil + return nil } // We have a digest, so we push use plain push for the digest. // Push here decides if it's a Manifest or a Blob. if err := repository.Push(ctx, d, reader); err != nil { - return nil, fmt.Errorf("failed to push: %w, %s", err, c.Ref) + return fmt.Errorf("failed to push: %w, %s", err, c.Ref) } - return &pushRequest{}, nil + return nil } func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadCloser, error) { @@ -131,7 +119,7 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadClose // that the mediatype is not set at this point. rdesc, err := src.Manifests().Resolve(ctx, desc.Digest.String()) if err != nil { - if strings.Contains(err.Error(), "not found") { + if errors.Is(err, oraserr.ErrNotFound) { rdesc, err = src.Blobs().Resolve(ctx, desc.Digest.String()) if err != nil { return nil, fmt.Errorf("failed to resolve fetch blob %q: %w", desc.Digest.String(), err) diff --git a/api/tech/oras/interface.go b/api/tech/oras/interface.go index ccd9ab1d4..5020f68d2 100644 --- a/api/tech/oras/interface.go +++ b/api/tech/oras/interface.go @@ -4,8 +4,6 @@ import ( "context" "io" - "github.com/containerd/containerd/content" - "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -54,22 +52,9 @@ type Fetcher interface { type Pusher interface { // Push returns a push request for the given resource identified // by the descriptor and the given data source. - Push(ctx context.Context, d ocispec.Descriptor, src Source) (PushRequest, error) + Push(ctx context.Context, d ocispec.Descriptor, src Source) error } type Lister interface { List(context.Context) ([]string, error) } - -// PushRequest handles the result of a push request -// replaces containerd content.Writer. -type PushRequest interface { - // Commit commits the blob (but no roll-back is guaranteed on an error). - // size and expected can be zero-value when unknown. - // Commit always closes the writer, even on error. - // ErrAlreadyExists aborts the writer. - Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error - - // Status returns the current state of write - Status() (content.Status, error) -} From 31c23e8facb3dc31b590ee30c12e276ac3b5543e Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:02:47 +0100 Subject: [PATCH 15/17] addressed feedback from PR review Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> - addressed sync lock missing to ensure concurrent calls - addressed code formatting issues - addressed missing Logger transport augmentation - addressed formatting and language issues - removed unused docker code --- .../repositories/ocireg/repository.go | 9 +- api/tech/docker/README.md | 6 - api/tech/docker/errors/errors.go | 60 -- api/tech/docker/fetcher.go | 203 ------ api/tech/docker/handler.go | 138 ---- api/tech/docker/httpreadseeker.go | 159 ----- api/tech/docker/lister.go | 131 ---- api/tech/docker/orig.go | 46 -- api/tech/docker/pusher.go | 434 ------------ api/tech/docker/registry.go | 236 ------- api/tech/docker/resolver.go | 657 ------------------ api/tech/oras/client.go | 59 +- api/tech/oras/delayed_reader.go | 15 +- go.mod | 2 - 14 files changed, 62 insertions(+), 2093 deletions(-) delete mode 100644 api/tech/docker/README.md delete mode 100644 api/tech/docker/errors/errors.go delete mode 100644 api/tech/docker/fetcher.go delete mode 100644 api/tech/docker/handler.go delete mode 100644 api/tech/docker/httpreadseeker.go delete mode 100644 api/tech/docker/lister.go delete mode 100644 api/tech/docker/orig.go delete mode 100644 api/tech/docker/pusher.go delete mode 100644 api/tech/docker/registry.go delete mode 100644 api/tech/docker/resolver.go diff --git a/api/oci/extensions/repositories/ocireg/repository.go b/api/oci/extensions/repositories/ocireg/repository.go index 391831533..1f1ede981 100644 --- a/api/oci/extensions/repositories/ocireg/repository.go +++ b/api/oci/extensions/repositories/ocireg/repository.go @@ -137,6 +137,7 @@ func (r *RepositoryImpl) getResolver(comp string) (oras.Resolver, error) { } client := retry.DefaultClient + client.Transport = ocmlog.NewRoundTripper(retry.DefaultClient.Transport, logger) if r.info.Scheme == "https" { // set up TLS //nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version. @@ -157,11 +158,9 @@ func (r *RepositoryImpl) getResolver(comp string) (oras.Resolver, error) { return rootCAs }(), } - client = &http.Client{ - Transport: retry.NewTransport(&http.Transport{ - TLSClientConfig: conf, - }), - } + client.Transport = ocmlog.NewRoundTripper(retry.NewTransport(&http.Transport{ + TLSClientConfig: conf, + }), logger) } authClient := &auth.Client{ diff --git a/api/tech/docker/README.md b/api/tech/docker/README.md deleted file mode 100644 index 7a6a1ae7f..000000000 --- a/api/tech/docker/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# containerd - -Taken from github.com/containerd/containerd remotes/docker to add list endpoints -Fix retry of requests with ResendBuffer - -This code is deprecated in favour of `regclient` package. diff --git a/api/tech/docker/errors/errors.go b/api/tech/docker/errors/errors.go deleted file mode 100644 index 71575a5da..000000000 --- a/api/tech/docker/errors/errors.go +++ /dev/null @@ -1,60 +0,0 @@ -//go:build deprecated - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package errors - -import ( - "fmt" - "io" - "net/http" -) - -var _ error = ErrUnexpectedStatus{} - -// ErrUnexpectedStatus is returned if a registry API request returned with unexpected HTTP status -type ErrUnexpectedStatus struct { - Status string - StatusCode int - Body []byte - RequestURL, RequestMethod string -} - -func (e ErrUnexpectedStatus) Error() string { - if len(e.Body) > 0 { - return fmt.Sprintf("unexpected status from %s request to %s: %s: %s", e.RequestMethod, e.RequestURL, e.Status, string(e.Body)) - } - return fmt.Sprintf("unexpected status from %s request to %s: %s", e.RequestMethod, e.RequestURL, e.Status) -} - -// NewUnexpectedStatusErr creates an ErrUnexpectedStatus from HTTP response -func NewUnexpectedStatusErr(resp *http.Response) error { - var b []byte - if resp.Body != nil { - b, _ = io.ReadAll(io.LimitReader(resp.Body, 64000)) // 64KB - } - err := ErrUnexpectedStatus{ - Body: b, - Status: resp.Status, - StatusCode: resp.StatusCode, - RequestMethod: resp.Request.Method, - } - if resp.Request.URL != nil { - err.RequestURL = resp.Request.URL.String() - } - return err -} diff --git a/api/tech/docker/fetcher.go b/api/tech/docker/fetcher.go deleted file mode 100644 index 4b325adac..000000000 --- a/api/tech/docker/fetcher.go +++ /dev/null @@ -1,203 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - - "ocm.software/ocm/api/utils/accessio" -) - -type dockerFetcher struct { - *dockerBase -} - -func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { - ctx = log.WithLogger(ctx, log.G(ctx).WithField("digest", desc.Digest)) - - hosts := r.filterHosts(HostCapabilityPull) - if len(hosts) == 0 { - return nil, errors.Wrap(errdefs.ErrNotFound, "no pull hosts") - } - - ctx, err := ContextWithRepositoryScope(ctx, r.refspec, false) - if err != nil { - return nil, err - } - - return newHTTPReadSeeker(desc.Size, func(offset int64) (io.ReadCloser, error) { - // firstly try fetch via external urls - for _, us := range desc.URLs { - ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", us)) - u, err := url.Parse(us) - if err != nil { - log.G(ctx).WithError(err).Debug("failed to parse") - continue - } - if u.Scheme != "http" && u.Scheme != "https" { - log.G(ctx).Debug("non-http(s) alternative url is unsupported") - continue - } - log.G(ctx).Debug("trying alternative url") - - // Try this first, parse it - host := RegistryHost{ - Client: http.DefaultClient, - Host: u.Host, - Scheme: u.Scheme, - Path: u.Path, - Capabilities: HostCapabilityPull, - } - req := r.request(host, http.MethodGet) - // Strip namespace from base - req.path = u.Path - if u.RawQuery != "" { - req.path = req.path + "?" + u.RawQuery - } - - rc, err := r.open(ctx, req, desc.MediaType, offset) - if err != nil { - if errdefs.IsNotFound(err) { - continue // try one of the other urls. - } - - return nil, err - } - - return rc, nil - } - - // Try manifests endpoints for manifests types - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, - images.MediaTypeDockerSchema1Manifest, - ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex: - - var firstErr error - for _, host := range r.hosts { - req := r.request(host, http.MethodGet, "manifests", desc.Digest.String()) - if err := req.addNamespace(r.refspec.Hostname()); err != nil { - return nil, err - } - - rc, err := r.open(ctx, req, desc.MediaType, offset) - if err != nil { - // Store the error for referencing later - if firstErr == nil { - firstErr = err - } - continue // try another host - } - - return rc, nil - } - - return nil, firstErr - } - - // Finally use blobs endpoints - var firstErr error - for _, host := range r.hosts { - req := r.request(host, http.MethodGet, "blobs", desc.Digest.String()) - if err := req.addNamespace(r.refspec.Hostname()); err != nil { - return nil, err - } - - rc, err := r.open(ctx, req, desc.MediaType, offset) - if err != nil { - // Store the error for referencing later - if firstErr == nil { - firstErr = err - } - continue // try another host - } - - return rc, nil - } - - if errdefs.IsNotFound(firstErr) { - firstErr = errors.Wrapf(errdefs.ErrNotFound, - "could not fetch content descriptor %v (%v) from remote", - desc.Digest, desc.MediaType) - } - - return nil, firstErr - }) -} - -func (r dockerFetcher) open(ctx context.Context, req *request, mediatype string, offset int64) (_ io.ReadCloser, retErr error) { - mt := "*/*" - if mediatype != "" { - mt = mediatype + ", " + mt - } - req.header.Set("Accept", mt) - - if offset > 0 { - // Note: "Accept-Ranges: bytes" cannot be trusted as some endpoints - // will return the header without supporting the range. The content - // range must always be checked. - req.header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) - } - - resp, err := req.doWithRetries(ctx, nil) - if err != nil { - return nil, accessio.RetriableError(err) - } - defer func() { - if retErr != nil { - resp.Body.Close() - } - }() - - if resp.StatusCode > 299 { - // TODO(stevvooe): When doing a offset specific request, we should - // really distinguish between a 206 and a 200. In the case of 200, we - // can discard the bytes, hiding the seek behavior from the - // implementation. - - if resp.StatusCode == http.StatusNotFound { - return nil, errors.Wrapf(errdefs.ErrNotFound, "content at %v not found", req.String()) - } - var registryErr Errors - if err := json.NewDecoder(resp.Body).Decode(®istryErr); err != nil || registryErr.Len() < 1 { - return nil, errors.Errorf("unexpected status code %v: %v", req.String(), resp.Status) - } - return nil, errors.Errorf("unexpected status code %v: %s - Server message: %s", req.String(), resp.Status, registryErr.Error()) - } - if offset > 0 { - cr := resp.Header.Get("content-range") - if cr != "" { - if !strings.HasPrefix(cr, fmt.Sprintf("bytes %d-", offset)) { - return nil, errors.Errorf("unhandled content range in response: %v", cr) - } - } else { - // TODO: Should any cases where use of content range - // without the proper header be considered? - // 206 responses? - - // Discard up to offset - // Could use buffer pool here but this case should be rare - n, err := io.Copy(io.Discard, io.LimitReader(resp.Body, offset)) - if err != nil { - return nil, errors.Wrap(err, "failed to discard to offset") - } - if n != offset { - return nil, errors.Errorf("unable to discard to offset") - } - } - } - - return resp.Body, nil -} diff --git a/api/tech/docker/handler.go b/api/tech/docker/handler.go deleted file mode 100644 index c761d6d0d..000000000 --- a/api/tech/docker/handler.go +++ /dev/null @@ -1,138 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "context" - "fmt" - "net/url" - "strings" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/labels" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/reference" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" -) - -// labelDistributionSource describes the source blob comes from. -var labelDistributionSource = "containerd.io/distribution.source" - -// AppendDistributionSourceLabel updates the label of blob with distribution source. -func AppendDistributionSourceLabel(manager content.Manager, ref string) (images.HandlerFunc, error) { - refspec, err := reference.Parse(ref) - if err != nil { - return nil, err - } - - u, err := url.Parse("dummy://" + refspec.Locator) - if err != nil { - return nil, err - } - - source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/") - return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - info, err := manager.Info(ctx, desc.Digest) - if err != nil { - return nil, err - } - - key := distributionSourceLabelKey(source) - - originLabel := "" - if info.Labels != nil { - originLabel = info.Labels[key] - } - value := appendDistributionSourceLabel(originLabel, repo) - - // The repo name has been limited under 256 and the distribution - // label might hit the limitation of label size, when blob data - // is used as the very, very common layer. - if err := labels.Validate(key, value); err != nil { - log.G(ctx).Warnf("skip to append distribution label: %s", err) - return nil, nil - } - - info = content.Info{ - Digest: desc.Digest, - Labels: map[string]string{ - key: value, - }, - } - _, err = manager.Update(ctx, info, fmt.Sprintf("labels.%s", key)) - return nil, err - }, nil -} - -func appendDistributionSourceLabel(originLabel, repo string) string { - repos := []string{} - if originLabel != "" { - repos = strings.Split(originLabel, ",") - } - repos = append(repos, repo) - - // use empty string to present duplicate items - for i := 1; i < len(repos); i++ { - tmp, j := repos[i], i-1 - for ; j >= 0 && repos[j] >= tmp; j-- { - if repos[j] == tmp { - tmp = "" - } - repos[j+1] = repos[j] - } - repos[j+1] = tmp - } - - i := 0 - for ; i < len(repos) && repos[i] == ""; i++ { - } - - return strings.Join(repos[i:], ",") -} - -func distributionSourceLabelKey(source string) string { - return fmt.Sprintf("%s.%s", labelDistributionSource, source) -} - -// selectRepositoryMountCandidate will select the repo which has longest -// common prefix components as the candidate. -func selectRepositoryMountCandidate(refspec reference.Spec, sources map[string]string) string { - u, err := url.Parse("dummy://" + refspec.Locator) - if err != nil { - // NOTE: basically, it won't be error here - return "" - } - - source, target := u.Hostname(), strings.TrimPrefix(u.Path, "/") - repoLabel, ok := sources[distributionSourceLabelKey(source)] - if !ok || repoLabel == "" { - return "" - } - - n, match := 0, "" - components := strings.Split(target, "/") - for _, repo := range strings.Split(repoLabel, ",") { - // the target repo is not a candidate - if repo == target { - continue - } - - if l := commonPrefixComponents(components, repo); l >= n { - n, match = l, repo - } - } - return match -} - -func commonPrefixComponents(components []string, target string) int { - targetComponents := strings.Split(target, "/") - - i := 0 - for ; i < len(components) && i < len(targetComponents); i++ { - if components[i] != targetComponents[i] { - break - } - } - return i -} diff --git a/api/tech/docker/httpreadseeker.go b/api/tech/docker/httpreadseeker.go deleted file mode 100644 index 374919b8d..000000000 --- a/api/tech/docker/httpreadseeker.go +++ /dev/null @@ -1,159 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "bytes" - "io" - - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/log" - "github.com/pkg/errors" -) - -const maxRetry = 3 - -type httpReadSeeker struct { - size int64 - offset int64 - rc io.ReadCloser - open func(offset int64) (io.ReadCloser, error) - closed bool - - errsWithNoProgress int -} - -func newHTTPReadSeeker(size int64, open func(offset int64) (io.ReadCloser, error)) (io.ReadCloser, error) { - return &httpReadSeeker{ - size: size, - open: open, - }, nil -} - -func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) { - if hrs.closed { - return 0, io.EOF - } - - rd, err := hrs.reader() - if err != nil { - return 0, err - } - - n, err = rd.Read(p) - hrs.offset += int64(n) - if n > 0 || err == nil { - hrs.errsWithNoProgress = 0 - } - - if !errors.Is(err, io.ErrUnexpectedEOF) { - return - } - // connection closed unexpectedly. try reconnecting. - if n == 0 { - hrs.errsWithNoProgress++ - if hrs.errsWithNoProgress > maxRetry { - return // too many retries for this offset with no progress - } - } - - if hrs.rc != nil { - if clsErr := hrs.rc.Close(); clsErr != nil { - log.L.WithError(clsErr).Errorf("httpReadSeeker: failed to close ReadCloser") - } - hrs.rc = nil - } - - if _, err2 := hrs.reader(); err2 == nil { - return n, nil - } - - return n, err -} - -func (hrs *httpReadSeeker) Close() error { - if hrs.closed { - return nil - } - hrs.closed = true - if hrs.rc != nil { - return hrs.rc.Close() - } - - return nil -} - -func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) { - if hrs.closed { - return 0, errors.Wrap(errdefs.ErrUnavailable, "Fetcher.Seek: closed") - } - - abs := hrs.offset - switch whence { - case io.SeekStart: - abs = offset - case io.SeekCurrent: - abs += offset - case io.SeekEnd: - if hrs.size == -1 { - return 0, errors.Wrap(errdefs.ErrUnavailable, "Fetcher.Seek: unknown size, cannot seek from end") - } - abs = hrs.size + offset - default: - return 0, errors.Wrap(errdefs.ErrInvalidArgument, "Fetcher.Seek: invalid whence") - } - - if abs < 0 { - return 0, errors.Wrapf(errdefs.ErrInvalidArgument, "Fetcher.Seek: negative offset") - } - - if abs != hrs.offset { - if hrs.rc != nil { - if err := hrs.rc.Close(); err != nil { - log.L.WithError(err).Errorf("Fetcher.Seek: failed to close ReadCloser") - } - - hrs.rc = nil - } - - hrs.offset = abs - } - - return hrs.offset, nil -} - -func (hrs *httpReadSeeker) reader() (io.Reader, error) { - if hrs.rc != nil { - return hrs.rc, nil - } - - if hrs.size == -1 || hrs.offset < hrs.size { - // only try to reopen the body request if we are seeking to a value - // less than the actual size. - if hrs.open == nil { - return nil, errors.Wrapf(errdefs.ErrNotImplemented, "cannot open") - } - - rc, err := hrs.open(hrs.offset) - if err != nil { - return nil, errors.Wrapf(err, "httpReadSeeker: failed open") - } - - if hrs.rc != nil { - if err := hrs.rc.Close(); err != nil { - log.L.WithError(err).Errorf("httpReadSeeker: failed to close ReadCloser") - } - } - hrs.rc = rc - } else { - // There is an edge case here where offset == size of the content. If - // we seek, we will probably get an error for content that cannot be - // sought (?). In that case, we should err on committing the content, - // as the length is already satisfied but we just return the empty - // reader instead. - - hrs.rc = io.NopCloser(bytes.NewReader([]byte{})) - } - - return hrs.rc, nil -} diff --git a/api/tech/docker/lister.go b/api/tech/docker/lister.go deleted file mode 100644 index aa492f3a7..000000000 --- a/api/tech/docker/lister.go +++ /dev/null @@ -1,131 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/log" - "github.com/pkg/errors" - "ocm.software/ocm/api/tech/oras" -) - -var ErrObjectNotRequired = errors.New("object not required") - -type TagList struct { - Name string `json:"name"` - Tags []string `json:"tags"` -} - -type dockerLister struct { - dockerBase *dockerBase -} - -func (r *dockerResolver) Lister(ctx context.Context, ref string) (oras.Lister, error) { - base, err := r.resolveDockerBase(ref) - if err != nil { - return nil, err - } - if base.refspec.Object != "" { - return nil, ErrObjectNotRequired - } - - return &dockerLister{ - dockerBase: base, - }, nil -} - -func (r *dockerLister) List(ctx context.Context) ([]string, error) { - refspec := r.dockerBase.refspec - base := r.dockerBase - var ( - firstErr error - paths [][]string - caps = HostCapabilityPull - ) - - // turns out, we have a valid digest, make a url. - paths = append(paths, []string{"tags/list"}) - caps |= HostCapabilityResolve - - hosts := base.filterHosts(caps) - if len(hosts) == 0 { - return nil, errors.Wrap(errdefs.ErrNotFound, "no list hosts") - } - - ctx, err := ContextWithRepositoryScope(ctx, refspec, false) - if err != nil { - return nil, err - } - - for _, u := range paths { - for _, host := range hosts { - ctxWithLogger := log.WithLogger(ctx, log.G(ctx).WithField("host", host.Host)) - - req := base.request(host, http.MethodGet, u...) - if err := req.addNamespace(base.refspec.Hostname()); err != nil { - return nil, err - } - - req.header["Accept"] = []string{"application/json"} - - log.G(ctxWithLogger).Debug("listing") - resp, err := req.doWithRetries(ctxWithLogger, nil) - if err != nil { - if errors.Is(err, ErrInvalidAuthorization) { - err = errors.Wrapf(err, "pull access denied, repository does not exist or may require authorization") - } - // Store the error for referencing later - if firstErr == nil { - firstErr = err - } - log.G(ctxWithLogger).WithError(err).Info("trying next host") - continue // try another host - } - - if resp.StatusCode > 299 { - resp.Body.Close() - if resp.StatusCode == http.StatusNotFound { - log.G(ctxWithLogger).Info("trying next host - response was http.StatusNotFound") - continue - } - if resp.StatusCode > 399 { - // Set firstErr when encountering the first non-404 status code. - if firstErr == nil { - firstErr = errors.Errorf("pulling from host %s failed with status code %v: %v", host.Host, u, resp.Status) - } - continue // try another host - } - return nil, errors.Errorf("taglist from host %s failed with unexpected status code %v: %v", host.Host, u, resp.Status) - } - - data, err := io.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, err - } - - tags := &TagList{} - - err = json.Unmarshal(data, tags) - if err != nil { - return nil, err - } - return tags.Tags, nil - } - } - - // If above loop terminates without return, then there was an error. - // "firstErr" contains the first non-404 error. That is, "firstErr == nil" - // means that either no registries were given or each registry returned 404. - - if firstErr == nil { - firstErr = errors.Wrap(errdefs.ErrNotFound, base.refspec.Locator) - } - - return nil, firstErr -} diff --git a/api/tech/docker/orig.go b/api/tech/docker/orig.go deleted file mode 100644 index 5ed0006d6..000000000 --- a/api/tech/docker/orig.go +++ /dev/null @@ -1,46 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "github.com/containerd/containerd/remotes/docker" -) - -var ( - ContextWithRepositoryScope = docker.ContextWithRepositoryScope - ContextWithAppendPullRepositoryScope = docker.ContextWithAppendPullRepositoryScope - NewInMemoryTracker = docker.NewInMemoryTracker - NewDockerAuthorizer = docker.NewDockerAuthorizer - WithAuthClient = docker.WithAuthClient - WithAuthHeader = docker.WithAuthHeader - WithAuthCreds = docker.WithAuthCreds -) - -type ( - Errors = docker.Errors - StatusTracker = docker.StatusTracker - Status = docker.Status - StatusTrackLocker = docker.StatusTrackLocker -) - -func ConvertHosts(hosts docker.RegistryHosts) RegistryHosts { - return func(host string) ([]RegistryHost, error) { - list, err := hosts(host) - if err != nil { - return nil, err - } - result := make([]RegistryHost, len(list)) - for i, v := range list { - result[i] = RegistryHost{ - Client: v.Client, - Authorizer: v.Authorizer, - Host: v.Host, - Scheme: v.Scheme, - Path: v.Path, - Capabilities: HostCapabilities(v.Capabilities), - Header: v.Header, - } - } - return result, nil - } -} diff --git a/api/tech/docker/pusher.go b/api/tech/docker/pusher.go deleted file mode 100644 index 0ab86dd4d..000000000 --- a/api/tech/docker/pusher.go +++ /dev/null @@ -1,434 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "context" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/remotes" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - remoteserrors "ocm.software/ocm/api/tech/docker/errors" - "ocm.software/ocm/api/tech/oras" - "ocm.software/ocm/api/utils/accessio" -) - -func init() { - l := logrus.New() - l.Level = logrus.WarnLevel - log.L = logrus.NewEntry(l) -} - -type dockerPusher struct { - *dockerBase - object string - - // TODO: namespace tracker - tracker StatusTracker -} - -func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor, src oras.Source) (oras.PushRequest, error) { - return p.push(ctx, desc, src, remotes.MakeRefKey(ctx, desc), false) -} - -func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, src oras.Source, ref string, unavailableOnFail bool) (oras.PushRequest, error) { - if l, ok := p.tracker.(StatusTrackLocker); ok { - l.Lock(ref) - defer l.Unlock(ref) - } - ctx, err := ContextWithRepositoryScope(ctx, p.refspec, true) - if err != nil { - return nil, err - } - status, err := p.tracker.GetStatus(ref) - if err == nil { - if status.Committed && status.Offset == status.Total { - return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "ref %v", ref) - } - if unavailableOnFail { - // Another push of this ref is happening elsewhere. The rest of function - // will continue only when `errdefs.IsNotFound(err) == true` (i.e. there - // is no actively-tracked ref already). - return nil, errors.Wrap(errdefs.ErrUnavailable, "push is on-going") - } - // TODO: Handle incomplete status - } else if !errdefs.IsNotFound(err) { - return nil, errors.Wrap(err, "failed to get status") - } - - hosts := p.filterHosts(HostCapabilityPush) - if len(hosts) == 0 { - return nil, errors.Wrap(errdefs.ErrNotFound, "no push hosts") - } - - var ( - isManifest bool - existCheck []string - host = hosts[0] - ) - - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, - ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex: - isManifest = true - existCheck = getManifestPath(p.object, desc.Digest) - default: - existCheck = []string{"blobs", desc.Digest.String()} - } - - req := p.request(host, http.MethodHead, existCheck...) - req.header.Set("Accept", strings.Join([]string{desc.MediaType, `*/*`}, ", ")) - - log.G(ctx).WithField("url", req.String()).Debugf("checking and pushing to") - - headResp, err := req.doWithRetries(ctx, nil) - if err != nil { - if !errors.Is(err, ErrInvalidAuthorization) { - return nil, err - } - log.G(ctx).WithError(err).Debugf("Unable to check existence, continuing with push") - } else { - defer headResp.Body.Close() - - if headResp.StatusCode == http.StatusOK { - var exists bool - if isManifest && existCheck[1] != desc.Digest.String() { - dgstHeader := digest.Digest(headResp.Header.Get("Docker-Content-Digest")) - if dgstHeader == desc.Digest { - exists = true - } - } else { - exists = true - } - - if exists { - p.tracker.SetStatus(ref, Status{ - Committed: true, - Status: content.Status{ - Ref: ref, - Total: desc.Size, - Offset: desc.Size, - // TODO: Set updated time? - }, - }) - - return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", desc.Digest) - } - } else if headResp.StatusCode != http.StatusNotFound { - err := remoteserrors.NewUnexpectedStatusErr(headResp) - - var statusError remoteserrors.ErrUnexpectedStatus - if errors.As(err, &statusError) { - log.G(ctx). - WithField("resp", headResp). - WithField("body", string(statusError.Body)). - Debug("unexpected response") - } - - return nil, accessio.RetriableError(err) - } - } - - if isManifest { - putPath := getManifestPath(p.object, desc.Digest) - req = p.request(host, http.MethodPut, putPath...) - req.header.Add("Content-Type", desc.MediaType) - } else { - // Start upload request - req = p.request(host, http.MethodPost, "blobs", "uploads/") - - var resp *http.Response - if fromRepo := selectRepositoryMountCandidate(p.refspec, desc.Annotations); fromRepo != "" { - preq := requestWithMountFrom(req, desc.Digest.String(), fromRepo) - pctx := ContextWithAppendPullRepositoryScope(ctx, fromRepo) - - // NOTE: the fromRepo might be private repo and - // auth service still can grant token without error. - // but the post request will fail because of 401. - // - // for the private repo, we should remove mount-from - // query and send the request again. - resp, err = preq.doWithRetries(pctx, nil) - if err != nil { - return nil, accessio.RetriableError(err) - } - - if resp.StatusCode == http.StatusUnauthorized { - log.G(ctx).Debugf("failed to mount from repository %s", fromRepo) - - resp.Body.Close() - resp = nil - } - } - - if resp == nil { - resp, err = req.doWithRetries(ctx, nil) - if err != nil { - return nil, accessio.RetriableError(err) - } - } - defer resp.Body.Close() - - switch resp.StatusCode { - case http.StatusOK, http.StatusAccepted, http.StatusNoContent: - case http.StatusCreated: - p.tracker.SetStatus(ref, Status{ - Committed: true, - Status: content.Status{ - Ref: ref, - Total: desc.Size, - Offset: desc.Size, - }, - }) - return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", desc.Digest) - default: - err := remoteserrors.NewUnexpectedStatusErr(resp) - - var statusError remoteserrors.ErrUnexpectedStatus - if errors.As(err, &statusError) { - log.G(ctx). - WithField("resp", resp). - WithField("body", string(statusError.Body)). - Debug("unexpected response") - } - - return nil, err - } - - var ( - location = resp.Header.Get("Location") - lurl *url.URL - lhost = host - ) - // Support paths without host in location - if strings.HasPrefix(location, "/") { - lurl, err = url.Parse(lhost.Scheme + "://" + lhost.Host + location) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse location %v", location) - } - } else { - if !strings.Contains(location, "://") { - location = lhost.Scheme + "://" + location - } - lurl, err = url.Parse(location) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse location %v", location) - } - - if lurl.Host != lhost.Host || lhost.Scheme != lurl.Scheme { - lhost.Scheme = lurl.Scheme - lhost.Host = lurl.Host - log.G(ctx).WithField("host", lhost.Host).WithField("scheme", lhost.Scheme).Debug("upload changed destination") - - // Strip authorizer if change to host or scheme - lhost.Authorizer = nil - } - } - q := lurl.Query() - q.Add("digest", desc.Digest.String()) - - req = p.request(lhost, http.MethodPut) - req.header.Set("Content-Type", "application/octet-stream") - req.path = lurl.Path + "?" + q.Encode() - } - p.tracker.SetStatus(ref, Status{ - Status: content.Status{ - Ref: ref, - Total: desc.Size, - Expected: desc.Digest, - StartedAt: time.Now(), - }, - }) - - // TODO: Support chunked upload - - respC := make(chan response, 1) - - preq := &pushRequest{ - base: p.dockerBase, - ref: ref, - responseC: respC, - source: src, - isManifest: isManifest, - expected: desc.Digest, - tracker: p.tracker, - } - - req.body = preq.Reader - req.size = desc.Size - - go func() { - defer close(respC) - resp, err := req.doWithRetries(ctx, nil) - if err != nil { - respC <- response{err: err} - return - } - - switch resp.StatusCode { - case http.StatusOK, http.StatusCreated, http.StatusNoContent: - default: - err := remoteserrors.NewUnexpectedStatusErr(resp) - - var statusError remoteserrors.ErrUnexpectedStatus - if errors.As(err, &statusError) { - log.G(ctx). - WithField("resp", resp). - WithField("body", string(statusError.Body)). - Debug("unexpected response") - } - } - respC <- response{Response: resp} - }() - - return preq, nil -} - -func getManifestPath(object string, dgst digest.Digest) []string { - if i := strings.IndexByte(object, '@'); i >= 0 { - if object[i+1:] != dgst.String() { - // use digest, not tag - object = "" - } else { - // strip @ for registry path to make tag - object = object[:i] - } - } - - if object == "" { - return []string{"manifests", dgst.String()} - } - - return []string{"manifests", object} -} - -type response struct { - *http.Response - err error -} - -type pushRequest struct { - base *dockerBase - ref string - - responseC <-chan response - source oras.Source - isManifest bool - - expected digest.Digest - tracker StatusTracker -} - -func (pw *pushRequest) Status() (content.Status, error) { - status, err := pw.tracker.GetStatus(pw.ref) - if err != nil { - return content.Status{}, err - } - return status.Status, nil -} - -func (pw *pushRequest) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { - // TODO: timeout waiting for response - resp := <-pw.responseC - if resp.err != nil { - return resp.err - } - defer resp.Response.Body.Close() - - // 201 is specified return status, some registries return - // 200, 202 or 204. - switch resp.StatusCode { - case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: - default: - return remoteserrors.NewUnexpectedStatusErr(resp.Response) - } - - status, err := pw.tracker.GetStatus(pw.ref) - if err != nil { - return errors.Wrap(err, "failed to get status") - } - - if size > 0 && size != status.Offset { - return errors.Errorf("unexpected size %d, expected %d", status.Offset, size) - } - - if expected == "" { - expected = status.Expected - } - - actual, err := digest.Parse(resp.Header.Get("Docker-Content-Digest")) - if err != nil { - return errors.Wrap(err, "invalid content digest in response") - } - - if actual != expected { - return errors.Errorf("got digest %s, expected %s", actual, expected) - } - - status.Committed = true - status.UpdatedAt = time.Now() - pw.tracker.SetStatus(pw.ref, status) - - return nil -} - -func (pw *pushRequest) Reader() (io.ReadCloser, error) { - status, err := pw.tracker.GetStatus(pw.ref) - if err != nil { - return nil, err - } - status.Offset = 0 - status.UpdatedAt = time.Now() - pw.tracker.SetStatus(pw.ref, status) - - r, err := pw.source.Reader() - if err != nil { - return nil, err - } - return &sizeTrackingReader{pw, r}, nil -} - -type sizeTrackingReader struct { - pw *pushRequest - io.ReadCloser -} - -func (t *sizeTrackingReader) Read(in []byte) (int, error) { - // fmt.Printf("reading next...\n") - n, err := t.ReadCloser.Read(in) - if n > 0 { - status, err := t.pw.tracker.GetStatus(t.pw.ref) - // fmt.Printf("read %d[%d] bytes\n", n, status.Offset) - if err != nil { - return n, err - } - status.Offset += int64(n) - status.UpdatedAt = time.Now() - t.pw.tracker.SetStatus(t.pw.ref, status) - } - return n, err -} - -func requestWithMountFrom(req *request, mount, from string) *request { - creq := *req - - sep := "?" - if strings.Contains(creq.path, sep) { - sep = "&" - } - - creq.path = creq.path + sep + "mount=" + mount + "&from=" + from - - return &creq -} diff --git a/api/tech/docker/registry.go b/api/tech/docker/registry.go deleted file mode 100644 index 90ed31483..000000000 --- a/api/tech/docker/registry.go +++ /dev/null @@ -1,236 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "net" - "net/http" - - "github.com/pkg/errors" -) - -// HostCapabilities represent the capabilities of the registry -// host. This also represents the set of operations for which -// the registry host may be trusted to perform. -// -// For example pushing is a capability which should only be -// performed on an upstream source, not a mirror. -// Resolving (the process of converting a name into a digest) -// must be considered a trusted operation and only done by -// a host which is trusted (or more preferably by secure process -// which can prove the provenance of the mapping). A public -// mirror should never be trusted to do a resolve action. -// -// | Registry Type | Pull | Resolve | Push | -// |------------------|------|---------|------| -// | Public Registry | yes | yes | yes | -// | Private Registry | yes | yes | yes | -// | Public Mirror | yes | no | no | -// | Private Mirror | yes | yes | no |. -type HostCapabilities uint8 - -const ( - // HostCapabilityPull represents the capability to fetch manifests - // and blobs by digest. - HostCapabilityPull HostCapabilities = 1 << iota - - // HostCapabilityResolve represents the capability to fetch manifests - // by name. - HostCapabilityResolve - - // HostCapabilityPush represents the capability to push blobs and - // manifests. - HostCapabilityPush - - // Reserved for future capabilities (i.e. search, catalog, remove). -) - -// Has checks whether the capabilities list has the provide capability. -func (c HostCapabilities) Has(t HostCapabilities) bool { - return c&t == t -} - -// RegistryHost represents a complete configuration for a registry -// host, representing the capabilities, authorizations, connection -// configuration, and location. -type RegistryHost struct { - Client *http.Client - Authorizer Authorizer - Host string - Scheme string - Path string - Capabilities HostCapabilities - Header http.Header -} - -const ( - dockerHostname = "docker.io" - dockerRegistryHostname = "registry-1.docker.io" -) - -func (h RegistryHost) isProxy(refhost string) bool { - if refhost != h.Host { - if refhost != dockerHostname || h.Host != dockerRegistryHostname { - return true - } - } - return false -} - -// RegistryHosts fetches the registry hosts for a given namespace, -// provided by the host component of an distribution image reference. -type RegistryHosts func(string) ([]RegistryHost, error) - -// Registries joins multiple registry configuration functions, using the same -// order as provided within the arguments. When an empty registry configuration -// is returned with a nil error, the next function will be called. -// NOTE: This function will not join configurations, as soon as a non-empty -// configuration is returned from a configuration function, it will be returned -// to the caller. -func Registries(registries ...RegistryHosts) RegistryHosts { - return func(host string) ([]RegistryHost, error) { - for _, registry := range registries { - config, err := registry(host) - if err != nil { - return config, err - } - if len(config) > 0 { - return config, nil - } - } - return nil, nil - } -} - -type registryOpts struct { - authorizer Authorizer - plainHTTP func(string) (bool, error) - host func(string) (string, error) - client *http.Client -} - -// RegistryOpt defines a registry default option. -type RegistryOpt func(*registryOpts) - -// WithPlainHTTP configures registries to use plaintext http scheme -// for the provided host match function. -func WithPlainHTTP(f func(string) (bool, error)) RegistryOpt { - return func(opts *registryOpts) { - opts.plainHTTP = f - } -} - -// WithAuthorizer configures the default authorizer for a registry. -func WithAuthorizer(a Authorizer) RegistryOpt { - return func(opts *registryOpts) { - opts.authorizer = a - } -} - -// WithHostTranslator defines the default translator to use for registry hosts. -func WithHostTranslator(h func(string) (string, error)) RegistryOpt { - return func(opts *registryOpts) { - opts.host = h - } -} - -// WithClient configures the default http client for a registry. -func WithClient(c *http.Client) RegistryOpt { - return func(opts *registryOpts) { - opts.client = c - } -} - -// ConfigureDefaultRegistries is used to create a default configuration for -// registries. For more advanced configurations or per-domain setups, -// the RegistryHosts interface should be used directly. -// NOTE: This function will always return a non-empty value or error. -func ConfigureDefaultRegistries(ropts ...RegistryOpt) RegistryHosts { - var opts registryOpts - for _, opt := range ropts { - opt(&opts) - } - - return func(host string) ([]RegistryHost, error) { - config := RegistryHost{ - Client: opts.client, - Authorizer: opts.authorizer, - Host: host, - Scheme: "https", - Path: "/v2", - Capabilities: HostCapabilityPull | HostCapabilityResolve | HostCapabilityPush, - } - - if config.Client == nil { - config.Client = http.DefaultClient - } - - if opts.plainHTTP != nil { - match, err := opts.plainHTTP(host) - if err != nil { - return nil, err - } - if match { - config.Scheme = "http" - } - } - - if opts.host != nil { - var err error - config.Host, err = opts.host(config.Host) - if err != nil { - return nil, err - } - } else if host == dockerHostname { - config.Host = dockerRegistryHostname - } - - return []RegistryHost{config}, nil - } -} - -// MatchAllHosts is a host match function which is always true. -func MatchAllHosts(string) (bool, error) { - return true, nil -} - -// MatchLocalhost is a host match function which returns true for -// localhost. -// -// Note: this does not handle matching of ip addresses in octal, -// decimal or hex form. -func MatchLocalhost(host string) (bool, error) { - switch { - case host == "::1": - return true, nil - case host == "[::1]": - return true, nil - } - h, p, err := net.SplitHostPort(host) - - // addrError helps distinguish between errors of form - // "no colon in address" and "too many colons in address". - // The former is fine as the host string need not have a - // port. Latter needs to be handled. - addrError := &net.AddrError{ - Err: "missing port in address", - Addr: host, - } - if err != nil { - if err.Error() != addrError.Error() { - return false, err - } - // host string without any port specified - h = host - } else if len(p) == 0 { - return false, errors.New("invalid host name format") - } - - // use ipv4 dotted decimal for further checking - if h == "localhost" { - h = "127.0.0.1" - } - ip := net.ParseIP(h) - - return ip.IsLoopback(), nil -} diff --git a/api/tech/docker/resolver.go b/api/tech/docker/resolver.go deleted file mode 100644 index 231a78976..000000000 --- a/api/tech/docker/resolver.go +++ /dev/null @@ -1,657 +0,0 @@ -//go:build deprecated - -package docker - -import ( - "context" - "fmt" - "io" - "net/http" - "net/url" - "path" - "strings" - - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/reference" - "github.com/containerd/containerd/remotes/docker/schema1" - "github.com/containerd/containerd/version" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/net/context/ctxhttp" - "ocm.software/ocm/api/tech/oras" - "ocm.software/ocm/api/utils/accessio" -) - -var ( - // ErrInvalidAuthorization is used when credentials are passed to a server but - // those credentials are rejected. - ErrInvalidAuthorization = errors.New("authorization failed") - - // MaxManifestSize represents the largest size accepted from a registry - // during resolution. Larger manifests may be accepted using a - // resolution method other than the registry. - // - // NOTE: The max supported layers by some runtimes is 128 and individual - // layers will not contribute more than 256 bytes, making a - // reasonable limit for a large image manifests of 32K bytes. - // 4M bytes represents a much larger upper bound for images which may - // contain large annotations or be non-images. A proper manifest - // design puts large metadata in subobjects, as is consistent the - // intent of the manifest design. - MaxManifestSize int64 = 4 * 1048 * 1048 -) - -// Authorizer is used to authorize HTTP requests based on 401 HTTP responses. -// An Authorizer is responsible for caching tokens or credentials used by -// requests. -type Authorizer interface { - // Authorize sets the appropriate `Authorization` header on the given - // request. - // - // If no authorization is found for the request, the request remains - // unmodified. It may also add an `Authorization` header as - // "bearer " - // "basic " - Authorize(context.Context, *http.Request) error - - // AddResponses adds a 401 response for the authorizer to consider when - // authorizing requests. The last response should be unauthorized and - // the previous requests are used to consider redirects and retries - // that may have led to the 401. - // - // If response is not handled, returns `ErrNotImplemented` - AddResponses(context.Context, []*http.Response) error -} - -// ResolverOptions are used to configured a new Docker register resolver. -type ResolverOptions struct { - // Hosts returns registry host configurations for a namespace. - Hosts RegistryHosts - - // Headers are the HTTP request header fields sent by the resolver - Headers http.Header - - // Tracker is used to track uploads to the registry. This is used - // since the registry does not have upload tracking and the existing - // mechanism for getting blob upload status is expensive. - Tracker StatusTracker - - // Authorizer is used to authorize registry requests - // Deprecated: use Hosts - Authorizer Authorizer - - // Credentials provides username and secret given a host. - // If username is empty but a secret is given, that secret - // is interpreted as a long lived token. - // Deprecated: use Hosts - Credentials func(string) (string, string, error) - - // Host provides the hostname given a namespace. - // Deprecated: use Hosts - Host func(string) (string, error) - - // PlainHTTP specifies to use plain http and not https - // Deprecated: use Hosts - PlainHTTP bool - - // Client is the http client to used when making registry requests - // Deprecated: use Hosts - Client *http.Client -} - -// DefaultHost is the default host function. -func DefaultHost(ns string) (string, error) { - if ns == "docker.io" { - return "registry-1.docker.io", nil - } - return ns, nil -} - -type dockerResolver struct { - hosts RegistryHosts - header http.Header - resolveHeader http.Header - tracker StatusTracker -} - -// NewResolver returns a new resolver to a Docker registry. -func NewResolver(options ResolverOptions) oras.Resolver { - if options.Tracker == nil { - options.Tracker = NewInMemoryTracker() - } - - if options.Headers == nil { - options.Headers = make(http.Header) - } - if _, ok := options.Headers["User-Agent"]; !ok { - options.Headers.Set("User-Agent", "containerd/"+version.Version) - } - - resolveHeader := http.Header{} - if _, ok := options.Headers["Accept"]; !ok { - // set headers for all the types we support for resolution. - resolveHeader.Set("Accept", strings.Join([]string{ - images.MediaTypeDockerSchema2Manifest, - images.MediaTypeDockerSchema2ManifestList, - ocispec.MediaTypeImageManifest, - ocispec.MediaTypeImageIndex, "*/*", - }, ", ")) - } else { - resolveHeader["Accept"] = options.Headers["Accept"] - delete(options.Headers, "Accept") - } - - if options.Hosts == nil { - opts := []RegistryOpt{} - if options.Host != nil { - opts = append(opts, WithHostTranslator(options.Host)) - } - - if options.Authorizer == nil { - options.Authorizer = NewDockerAuthorizer( - WithAuthClient(options.Client), - WithAuthHeader(options.Headers), - WithAuthCreds(options.Credentials)) - } - opts = append(opts, WithAuthorizer(options.Authorizer)) - - if options.Client != nil { - opts = append(opts, WithClient(options.Client)) - } - if options.PlainHTTP { - opts = append(opts, WithPlainHTTP(MatchAllHosts)) - } else { - opts = append(opts, WithPlainHTTP(MatchLocalhost)) - } - options.Hosts = ConfigureDefaultRegistries(opts...) - } - return &dockerResolver{ - hosts: options.Hosts, - header: options.Headers, - resolveHeader: resolveHeader, - tracker: options.Tracker, - } -} - -func getManifestMediaType(resp *http.Response) string { - // Strip encoding data (manifests should always be ascii JSON) - contentType := resp.Header.Get("Content-Type") - if sp := strings.IndexByte(contentType, ';'); sp != -1 { - contentType = contentType[0:sp] - } - - // As of Apr 30 2019 the registry.access.redhat.com registry does not specify - // the content type of any data but uses schema1 manifests. - if contentType == "text/plain" { - contentType = images.MediaTypeDockerSchema1Manifest - } - return contentType -} - -type countingReader struct { - reader io.Reader - bytesRead int64 -} - -func (r *countingReader) Read(p []byte) (int, error) { - n, err := r.reader.Read(p) - r.bytesRead += int64(n) - return n, err -} - -var _ oras.Resolver = &dockerResolver{} - -func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) { - base, err := r.resolveDockerBase(ref) - if err != nil { - return "", ocispec.Descriptor{}, err - } - refspec := base.refspec - if refspec.Object == "" { - return "", ocispec.Descriptor{}, reference.ErrObjectRequired - } - - var ( - firstErr error - paths [][]string - dgst = refspec.Digest() - caps = HostCapabilityPull - ) - - if dgst != "" { - if err := dgst.Validate(); err != nil { - // need to fail here, since we can't actually resolve the invalid - // digest. - return "", ocispec.Descriptor{}, err - } - - // turns out, we have a valid digest, make a url. - paths = append(paths, []string{"manifests", dgst.String()}) - - // fallback to blobs on not found. - paths = append(paths, []string{"blobs", dgst.String()}) - } else { - // Add - paths = append(paths, []string{"manifests", refspec.Object}) - caps |= HostCapabilityResolve - } - - hosts := base.filterHosts(caps) - if len(hosts) == 0 { - return "", ocispec.Descriptor{}, errors.Wrap(errdefs.ErrNotFound, "no resolve hosts") - } - - ctx, err = ContextWithRepositoryScope(ctx, refspec, false) - if err != nil { - return "", ocispec.Descriptor{}, err - } - - for _, u := range paths { - for _, host := range hosts { - ctxWithLogger := log.WithLogger(ctx, log.G(ctx).WithField("host", host.Host)) - - req := base.request(host, http.MethodHead, u...) - if err := req.addNamespace(base.refspec.Hostname()); err != nil { - return "", ocispec.Descriptor{}, err - } - - for key, value := range r.resolveHeader { - req.header[key] = append(req.header[key], value...) - } - - log.G(ctxWithLogger).Debug("resolving") - resp, err := req.doWithRetries(ctxWithLogger, nil) - if err != nil { - if errors.Is(err, ErrInvalidAuthorization) { - err = errors.Wrapf(err, "pull access denied, repository does not exist or may require authorization") - } else { - err = accessio.RetriableError(err) - } - // Store the error for referencing later - if firstErr == nil { - firstErr = err - } - log.G(ctxWithLogger).WithError(err).Info("trying next host") - continue // try another host - } - resp.Body.Close() // don't care about body contents. - - if resp.StatusCode > 299 { - if resp.StatusCode == http.StatusNotFound { - // log.G(ctxWithLogger).Info("trying next host - response was http.StatusNotFound") - continue - } - if resp.StatusCode > 399 { - // Set firstErr when encountering the first non-404 status code. - if firstErr == nil { - firstErr = errors.Errorf("pulling from host %s failed with status code %v: %v", host.Host, u, resp.Status) - } - continue // try another host - } - return "", ocispec.Descriptor{}, errors.Errorf("pulling from host %s failed with unexpected status code %v: %v", host.Host, u, resp.Status) - } - size := resp.ContentLength - contentType := getManifestMediaType(resp) - - // if no digest was provided, then only a resolve - // trusted registry was contacted, in this case use - // the digest header (or content from GET) - if dgst == "" { - // this is the only point at which we trust the registry. we use the - // content headers to assemble a descriptor for the name. when this becomes - // more robust, we mostly get this information from a secure trust store. - dgstHeader := digest.Digest(resp.Header.Get("Docker-Content-Digest")) - - if dgstHeader != "" && size != -1 { - if err := dgstHeader.Validate(); err != nil { - return "", ocispec.Descriptor{}, errors.Wrapf(err, "%q in header not a valid digest", dgstHeader) - } - dgst = dgstHeader - } - } - if dgst == "" || size == -1 { - log.G(ctxWithLogger).Debug("no Docker-Content-Digest header, fetching manifest instead") - - req = base.request(host, http.MethodGet, u...) - if err := req.addNamespace(base.refspec.Hostname()); err != nil { - return "", ocispec.Descriptor{}, err - } - - for key, value := range r.resolveHeader { - req.header[key] = append(req.header[key], value...) - } - - resp, err := req.doWithRetries(ctxWithLogger, nil) - if err != nil { - return "", ocispec.Descriptor{}, accessio.RetriableError(err) - } - defer resp.Body.Close() - - bodyReader := countingReader{reader: resp.Body} - - contentType = getManifestMediaType(resp) - if dgst == "" { - if contentType == images.MediaTypeDockerSchema1Manifest { - b, err := schema1.ReadStripSignature(&bodyReader) - if err != nil { - return "", ocispec.Descriptor{}, accessio.RetriableError(err) - } - - dgst = digest.FromBytes(b) - } else { - dgst, err = digest.FromReader(&bodyReader) - if err != nil { - return "", ocispec.Descriptor{}, accessio.RetriableError(err) - } - } - } else if _, err := io.Copy(io.Discard, &bodyReader); err != nil { - return "", ocispec.Descriptor{}, accessio.RetriableError(err) - } - size = bodyReader.bytesRead - } - // Prevent resolving to excessively large manifests - if size > MaxManifestSize { - if firstErr == nil { - firstErr = errors.Wrapf(errdefs.ErrNotFound, "rejecting %d byte manifest for %s", size, ref) - } - continue - } - - desc := ocispec.Descriptor{ - Digest: dgst, - MediaType: contentType, - Size: size, - } - - log.G(ctxWithLogger).WithField("desc.digest", desc.Digest).Debug("resolved") - return ref, desc, nil - } - } - - // If above loop terminates without return, then there was an error. - // "firstErr" contains the first non-404 error. That is, "firstErr == nil" - // means that either no registries were given or each registry returned 404. - - if firstErr == nil { - firstErr = errors.Wrap(errdefs.ErrNotFound, ref) - } - - return "", ocispec.Descriptor{}, firstErr -} - -func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (oras.Fetcher, error) { - base, err := r.resolveDockerBase(ref) - if err != nil { - return nil, err - } - - return dockerFetcher{ - dockerBase: base, - }, nil -} - -func (r *dockerResolver) Pusher(ctx context.Context, ref string) (oras.Pusher, error) { - base, err := r.resolveDockerBase(ref) - if err != nil { - return nil, err - } - - return dockerPusher{ - dockerBase: base, - object: base.refspec.Object, - tracker: r.tracker, - }, nil -} - -func (r *dockerResolver) resolveDockerBase(ref string) (*dockerBase, error) { - refspec, err := reference.Parse(ref) - if err != nil { - return nil, err - } - - return r.base(refspec) -} - -type dockerBase struct { - refspec reference.Spec - repository string - hosts []RegistryHost - header http.Header -} - -func (r *dockerResolver) base(refspec reference.Spec) (*dockerBase, error) { - host := refspec.Hostname() - hosts, err := r.hosts(host) - if err != nil { - return nil, err - } - return &dockerBase{ - refspec: refspec, - repository: strings.TrimPrefix(refspec.Locator, host+"/"), - hosts: hosts, - header: r.header, - }, nil -} - -func (r *dockerBase) filterHosts(caps HostCapabilities) (hosts []RegistryHost) { - for _, host := range r.hosts { - if host.Capabilities.Has(caps) { - hosts = append(hosts, host) - } - } - return -} - -func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *request { - header := r.header.Clone() - if header == nil { - header = http.Header{} - } - - for key, value := range host.Header { - header[key] = append(header[key], value...) - } - parts := append([]string{"/", host.Path, r.repository}, ps...) - p := path.Join(parts...) - // Join strips trailing slash, re-add ending "/" if included - if len(parts) > 0 && strings.HasSuffix(parts[len(parts)-1], "/") { - p += "/" - } - return &request{ - method: method, - path: p, - header: header, - host: host, - } -} - -func (r *request) authorize(ctx context.Context, req *http.Request) error { - // Check if has header for host - if r.host.Authorizer != nil { - if err := r.host.Authorizer.Authorize(ctx, req); err != nil { - return err - } - } - - return nil -} - -func (r *request) addNamespace(ns string) (err error) { - if !r.host.isProxy(ns) { - return nil - } - var q url.Values - // Parse query - if i := strings.IndexByte(r.path, '?'); i > 0 { - r.path = r.path[:i+1] - q, err = url.ParseQuery(r.path[i+1:]) - if err != nil { - return - } - } else { - r.path += "?" - q = url.Values{} - } - q.Add("ns", ns) - - r.path += q.Encode() - - return -} - -type request struct { - method string - path string - header http.Header - host RegistryHost - body func() (io.ReadCloser, error) - size int64 -} - -func (r *request) do(ctx context.Context) (*http.Response, error) { - u := r.host.Scheme + "://" + r.host.Host + r.path - req, err := http.NewRequestWithContext(ctx, r.method, u, nil) - if err != nil { - return nil, err - } - req.Header = http.Header{} // headers need to be copied to avoid concurrent map access - for k, v := range r.header { - req.Header[k] = v - } - if r.body != nil { - body, err := r.body() - if err != nil { - return nil, err - } - req.Body = body - req.GetBody = r.body - if r.size > 0 { - req.ContentLength = r.size - } - defer body.Close() - } - - ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", u)) - log.G(ctx).WithFields(sanitizedRequestFields(req)).Debug("do request") - if err := r.authorize(ctx, req); err != nil { - return nil, errors.Wrap(err, "failed to authorize") - } - - client := &http.Client{} - if r.host.Client != nil { - *client = *r.host.Client - } - if client.CheckRedirect == nil { - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - if len(via) >= 10 { - return errors.New("stopped after 10 redirects") - } - return errors.Wrap(r.authorize(ctx, req), "failed to authorize redirect") - } - } - - resp, err := ctxhttp.Do(ctx, client, req) - if err != nil { - return nil, errors.Wrap(err, "failed to do request") - } - log.G(ctx).WithFields(responseFields(resp)).Debug("fetch response received") - return resp, nil -} - -func (r *request) doWithRetries(ctx context.Context, responses []*http.Response) (*http.Response, error) { - resp, err := r.do(ctx) - if err != nil { - return nil, err - } - - responses = append(responses, resp) - retry, err := r.retryRequest(ctx, responses) - if err != nil { - resp.Body.Close() - return nil, err - } - if retry { - resp.Body.Close() - return r.doWithRetries(ctx, responses) - } - return resp, err -} - -func (r *request) retryRequest(ctx context.Context, responses []*http.Response) (bool, error) { - if len(responses) > 5 { - return false, nil - } - last := responses[len(responses)-1] - switch last.StatusCode { - case http.StatusUnauthorized: - log.G(ctx).WithField("header", last.Header.Get("WWW-Authenticate")).Debug("Unauthorized") - if r.host.Authorizer != nil { - if err := r.host.Authorizer.AddResponses(ctx, responses); err == nil { - return true, nil - } else if !errdefs.IsNotImplemented(err) { - return false, err - } - } - - return false, nil - case http.StatusMethodNotAllowed: - // Support registries which have not properly implemented the HEAD method for - // manifests endpoint - if r.method == http.MethodHead && strings.Contains(r.path, "/manifests/") { - r.method = http.MethodGet - return true, nil - } - case http.StatusRequestTimeout, http.StatusTooManyRequests: - return true, nil - } - - // TODO: Handle 50x errors accounting for attempt history - return false, nil -} - -func (r *request) String() string { - return r.host.Scheme + "://" + r.host.Host + r.path -} - -func sanitizedRequestFields(req *http.Request) logrus.Fields { - fields := map[string]interface{}{ - "request.method": req.Method, - } - for k, vals := range req.Header { - k = strings.ToLower(k) - if k == "authorization" { - continue - } - for i, v := range vals { - field := "request.header." + k - if i > 0 { - field = fmt.Sprintf("%s.%d", field, i) - } - fields[field] = v - } - } - - return logrus.Fields(fields) -} - -func responseFields(resp *http.Response) logrus.Fields { - fields := map[string]interface{}{ - "response.status": resp.Status, - } - for k, vals := range resp.Header { - k = strings.ToLower(k) - for i, v := range vals { - field := "response.header." + k - if i > 0 { - field = fmt.Sprintf("%s.%d", field, i) - } - fields[field] = v - } - } - - return logrus.Fields(fields) -} diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index 6ce26019c..4032c0424 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "strings" + "sync" "github.com/containerd/containerd/errdefs" ociv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -23,6 +24,8 @@ type Client struct { Client *auth.Client PlainHTTP bool Ref string + + rw sync.Mutex } var ( @@ -37,6 +40,9 @@ func New(opts ClientOptions) *Client { } func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { + c.rw.Lock() + defer c.rw.Unlock() + src, err := c.createRepository(ref) if err != nil { return "", ociv1.Descriptor{}, err @@ -60,21 +66,33 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip } func (c *Client) Fetcher(ctx context.Context, ref string) (Fetcher, error) { + c.rw.Lock() + defer c.rw.Unlock() + c.Ref = ref return c, nil } func (c *Client) Pusher(ctx context.Context, ref string) (Pusher, error) { + c.rw.Lock() + defer c.rw.Unlock() + c.Ref = ref return c, nil } func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { + c.rw.Lock() + defer c.rw.Unlock() + c.Ref = ref return c, nil } func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error { + c.rw.Lock() + defer c.rw.Unlock() + reader, err := src.Reader() if err != nil { return err @@ -97,7 +115,7 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error return nil } - // We have a digest, so we push use plain push for the digest. + // We have a digest, so we use plain push for the digest. // Push here decides if it's a Manifest or a Blob. if err := repository.Push(ctx, d, reader); err != nil { return fmt.Errorf("failed to push: %w, %s", err, c.Ref) @@ -107,30 +125,36 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error } func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadCloser, error) { + c.rw.Lock() + defer c.rw.Unlock() + src, err := c.createRepository(c.Ref) if err != nil { return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) } // oras requires a Resolve to happen before a fetch because - // -1 is an invalid size. + // -1 is an invalid size and results in a content-length mismatch error by design. + // This is a security consideration on ORAS' side. // manifest is not set in the descriptor // We explicitly call resolve on manifest first because it might be - // that the mediatype is not set at this point. + // that the mediatype is not set at this point so we don't want ORAS to try to + // select the wrong layer to fetch from. rdesc, err := src.Manifests().Resolve(ctx, desc.Digest.String()) - if err != nil { - if errors.Is(err, oraserr.ErrNotFound) { - rdesc, err = src.Blobs().Resolve(ctx, desc.Digest.String()) - if err != nil { - return nil, fmt.Errorf("failed to resolve fetch blob %q: %w", desc.Digest.String(), err) - } - delayer := func() (io.ReadCloser, error) { - return src.Blobs().Fetch(ctx, rdesc) - } - - return newDelayedReader(delayer) + if errors.Is(err, oraserr.ErrNotFound) { + rdesc, err = src.Blobs().Resolve(ctx, desc.Digest.String()) + if err != nil { + return nil, fmt.Errorf("failed to resolve fetch blob %q: %w", desc.Digest.String(), err) } + delayer := func() (io.ReadCloser, error) { + return src.Blobs().Fetch(ctx, rdesc) + } + + return newDelayedReader(delayer) + } + + if err != nil { return nil, fmt.Errorf("failed to resolve fetch manifest %q: %w", desc.Digest.String(), err) } @@ -144,12 +168,15 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadClose } func (c *Client) List(ctx context.Context) ([]string, error) { - var result []string + c.rw.Lock() + defer c.rw.Unlock() + src, err := c.createRepository(c.Ref) if err != nil { return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) } + var result []string if err := src.Tags(ctx, "", func(tags []string) error { result = append(result, tags...) return nil @@ -160,6 +187,8 @@ func (c *Client) List(ctx context.Context) ([]string, error) { return result, nil } +// createRepository creates a new repository representation using the passed in ref. +// This is a cheap operation. func (c *Client) createRepository(ref string) (*remote.Repository, error) { src, err := remote.NewRepository(ref) if err != nil { diff --git a/api/tech/oras/delayed_reader.go b/api/tech/oras/delayed_reader.go index 9918b78c5..abc3acdac 100644 --- a/api/tech/oras/delayed_reader.go +++ b/api/tech/oras/delayed_reader.go @@ -1,6 +1,9 @@ package oras -import "io" +import ( + "io" + "sync" +) // delayedReader sets up a reader that only fetches a blob // upon explicit reading request, otherwise, it stores the @@ -9,6 +12,7 @@ type delayedReader struct { open func() (io.ReadCloser, error) rc io.ReadCloser closed bool + rw sync.Mutex } func newDelayedReader(open func() (io.ReadCloser, error)) (*delayedReader, error) { @@ -18,6 +22,9 @@ func newDelayedReader(open func() (io.ReadCloser, error)) (*delayedReader, error } func (d *delayedReader) Read(p []byte) (n int, err error) { + d.rw.Lock() + defer d.rw.Unlock() + if d.closed { return 0, io.EOF } @@ -31,6 +38,9 @@ func (d *delayedReader) Read(p []byte) (n int, err error) { } func (d *delayedReader) reader() (io.ReadCloser, error) { + d.rw.Lock() + defer d.rw.Unlock() + if d.rc != nil { return d.rc, nil } @@ -45,6 +55,9 @@ func (d *delayedReader) reader() (io.ReadCloser, error) { } func (d *delayedReader) Close() error { + d.rw.Lock() + defer d.rw.Unlock() + if d.closed { return nil } diff --git a/go.mod b/go.mod index e2c53c704..10e02fada 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,6 @@ go 1.23.2 replace github.com/spf13/cobra => github.com/open-component-model/cobra v0.0.0-20230329075350-b1fd876abfb9 -//replace github.com/regclient/regclient => /Users/skarlso/goprojects/regclient - require ( dario.cat/mergo v1.0.1 github.com/DataDog/gostackparse v0.7.0 From 934f789744b43a5553362d01cdc4d623fd6ea1f4 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:16:47 +0100 Subject: [PATCH 16/17] unexport the two fields in client and remove the mutext from delayed reader --- api/tech/oras/client.go | 65 ++++++++++++++++----------------- api/tech/oras/delayed_reader.go | 11 ------ 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index 4032c0424..76138dc39 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -21,11 +21,10 @@ type ClientOptions struct { } type Client struct { - Client *auth.Client - PlainHTTP bool - Ref string - - rw sync.Mutex + client *auth.Client + plainHTTP bool + ref string + mu sync.Mutex } var ( @@ -36,12 +35,12 @@ var ( ) func New(opts ClientOptions) *Client { - return &Client{Client: opts.Client, PlainHTTP: opts.PlainHTTP} + return &Client{client: opts.Client, plainHTTP: opts.PlainHTTP} } func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { - c.rw.Lock() - defer c.rw.Unlock() + c.mu.Lock() + defer c.mu.Unlock() src, err := c.createRepository(ref) if err != nil { @@ -66,49 +65,49 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip } func (c *Client) Fetcher(ctx context.Context, ref string) (Fetcher, error) { - c.rw.Lock() - defer c.rw.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - c.Ref = ref + c.ref = ref return c, nil } func (c *Client) Pusher(ctx context.Context, ref string) (Pusher, error) { - c.rw.Lock() - defer c.rw.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - c.Ref = ref + c.ref = ref return c, nil } func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { - c.rw.Lock() - defer c.rw.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - c.Ref = ref + c.ref = ref return c, nil } func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error { - c.rw.Lock() - defer c.rw.Unlock() + c.mu.Lock() + defer c.mu.Unlock() reader, err := src.Reader() if err != nil { return err } - repository, err := c.createRepository(c.Ref) + repository, err := c.createRepository(c.ref) if err != nil { return err } - if split := strings.Split(c.Ref, ":"); len(split) == 2 { + if split := strings.Split(c.ref, ":"); len(split) == 2 { // Once we get a reference that contains a tag, we need to re-push that // layer with the reference included. PushReference then will tag // that layer resulting in the created tag pointing to the right // blob data. - if err := repository.PushReference(ctx, d, reader, c.Ref); err != nil { + if err := repository.PushReference(ctx, d, reader, c.ref); err != nil { return fmt.Errorf("failed to push tag: %w", err) } @@ -118,19 +117,19 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error // We have a digest, so we use plain push for the digest. // Push here decides if it's a Manifest or a Blob. if err := repository.Push(ctx, d, reader); err != nil { - return fmt.Errorf("failed to push: %w, %s", err, c.Ref) + return fmt.Errorf("failed to push: %w, %s", err, c.ref) } return nil } func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadCloser, error) { - c.rw.Lock() - defer c.rw.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - src, err := c.createRepository(c.Ref) + src, err := c.createRepository(c.ref) if err != nil { - return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) + return nil, fmt.Errorf("failed to resolve ref %q: %w", c.ref, err) } // oras requires a Resolve to happen before a fetch because @@ -168,12 +167,12 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadClose } func (c *Client) List(ctx context.Context) ([]string, error) { - c.rw.Lock() - defer c.rw.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - src, err := c.createRepository(c.Ref) + src, err := c.createRepository(c.ref) if err != nil { - return nil, fmt.Errorf("failed to resolve ref %q: %w", c.Ref, err) + return nil, fmt.Errorf("failed to resolve ref %q: %w", c.ref, err) } var result []string @@ -195,8 +194,8 @@ func (c *Client) createRepository(ref string) (*remote.Repository, error) { return nil, fmt.Errorf("failed to create new repository: %w", err) } - src.Client = c.Client // set up authenticated client. - src.PlainHTTP = c.PlainHTTP + src.Client = c.client // set up authenticated client. + src.PlainHTTP = c.plainHTTP return src, nil } diff --git a/api/tech/oras/delayed_reader.go b/api/tech/oras/delayed_reader.go index abc3acdac..d07a6f870 100644 --- a/api/tech/oras/delayed_reader.go +++ b/api/tech/oras/delayed_reader.go @@ -2,7 +2,6 @@ package oras import ( "io" - "sync" ) // delayedReader sets up a reader that only fetches a blob @@ -12,7 +11,6 @@ type delayedReader struct { open func() (io.ReadCloser, error) rc io.ReadCloser closed bool - rw sync.Mutex } func newDelayedReader(open func() (io.ReadCloser, error)) (*delayedReader, error) { @@ -22,9 +20,6 @@ func newDelayedReader(open func() (io.ReadCloser, error)) (*delayedReader, error } func (d *delayedReader) Read(p []byte) (n int, err error) { - d.rw.Lock() - defer d.rw.Unlock() - if d.closed { return 0, io.EOF } @@ -38,9 +33,6 @@ func (d *delayedReader) Read(p []byte) (n int, err error) { } func (d *delayedReader) reader() (io.ReadCloser, error) { - d.rw.Lock() - defer d.rw.Unlock() - if d.rc != nil { return d.rc, nil } @@ -55,9 +47,6 @@ func (d *delayedReader) reader() (io.ReadCloser, error) { } func (d *delayedReader) Close() error { - d.rw.Lock() - defer d.rw.Unlock() - if d.closed { return nil } From 43ed229ccff7218c98ff59a6d55a0bb6f7734ef1 Mon Sep 17 00:00:00 2001 From: Gergely Brautigam <182850+Skarlso@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:43:03 +0100 Subject: [PATCH 17/17] fix: replace the lock with an RW lock to only lock the read on list and fetch --- api/tech/oras/client.go | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/api/tech/oras/client.go b/api/tech/oras/client.go index 76138dc39..a73e0003a 100644 --- a/api/tech/oras/client.go +++ b/api/tech/oras/client.go @@ -24,7 +24,7 @@ type Client struct { client *auth.Client plainHTTP bool ref string - mu sync.Mutex + mu sync.RWMutex } var ( @@ -38,10 +38,34 @@ func New(opts ClientOptions) *Client { return &Client{client: opts.Client, plainHTTP: opts.PlainHTTP} } -func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { +func (c *Client) Fetcher(ctx context.Context, ref string) (Fetcher, error) { + c.mu.Lock() + defer c.mu.Unlock() + + c.ref = ref + return c, nil +} + +func (c *Client) Pusher(ctx context.Context, ref string) (Pusher, error) { c.mu.Lock() defer c.mu.Unlock() + c.ref = ref + return c, nil +} + +func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { + c.mu.Lock() + defer c.mu.Unlock() + + c.ref = ref + return c, nil +} + +func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descriptor, error) { + c.mu.RLock() + defer c.mu.RUnlock() + src, err := c.createRepository(ref) if err != nil { return "", ociv1.Descriptor{}, err @@ -64,33 +88,9 @@ func (c *Client) Resolve(ctx context.Context, ref string) (string, ociv1.Descrip return "", desc, nil } -func (c *Client) Fetcher(ctx context.Context, ref string) (Fetcher, error) { - c.mu.Lock() - defer c.mu.Unlock() - - c.ref = ref - return c, nil -} - -func (c *Client) Pusher(ctx context.Context, ref string) (Pusher, error) { - c.mu.Lock() - defer c.mu.Unlock() - - c.ref = ref - return c, nil -} - -func (c *Client) Lister(ctx context.Context, ref string) (Lister, error) { - c.mu.Lock() - defer c.mu.Unlock() - - c.ref = ref - return c, nil -} - func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() reader, err := src.Reader() if err != nil { @@ -124,8 +124,8 @@ func (c *Client) Push(ctx context.Context, d ociv1.Descriptor, src Source) error } func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadCloser, error) { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() src, err := c.createRepository(c.ref) if err != nil { @@ -167,8 +167,8 @@ func (c *Client) Fetch(ctx context.Context, desc ociv1.Descriptor) (io.ReadClose } func (c *Client) List(ctx context.Context) ([]string, error) { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() src, err := c.createRepository(c.ref) if err != nil {