Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: replace docker client code with oras - take 2 #1184

Merged
merged 20 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
82c5607
feat: replace docker client code with regclient
Skarlso Dec 10, 2024
7dd6512
fix the nil and the unit test
Skarlso Dec 10, 2024
1b51dc4
updated internal ref and the way we resolve manifests
Skarlso Dec 11, 2024
cc7c285
fix fetching
Skarlso Dec 11, 2024
cd8281b
fix fetching
Skarlso Dec 12, 2024
8ad52af
using heads
Skarlso Dec 12, 2024
a51766c
added delayed reader and blob head check
Skarlso Dec 12, 2024
b3911c9
add deprecation tags on docker code and add returning not found in cl…
Skarlso Dec 12, 2024
cc3c741
Merge branch 'main' into replace-docker-with-regclient
jakobmoellerdev Dec 16, 2024
8287d45
trying out the entire flow with ORAS instead
Skarlso Dec 16, 2024
2835b38
Merge branch 'replace-docker-with-regclient' of github.com:open-compo…
Skarlso Dec 16, 2024
7358582
removed fmt.Println
Skarlso Dec 16, 2024
f7ce457
fixin some lintin
Skarlso Dec 16, 2024
c9e2830
renamed the createRepository function and fixed digest resolve
Skarlso Dec 16, 2024
78d2bc8
remove regctl and create the right oras client
Skarlso Dec 16, 2024
024a606
simplified the interface and removed the unused status and commit
Skarlso Dec 16, 2024
31c23e8
addressed feedback from PR review
Skarlso Dec 17, 2024
934f789
unexport the two fields in client and remove the mutext from delayed …
Skarlso Dec 17, 2024
43ed229
fix: replace the lock with an RW lock to only lock the read on list a…
Skarlso Dec 18, 2024
086d16b
Merge branch 'main' into replace-docker-with-regclient
jakobmoellerdev Dec 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions api/oci/extensions/repositories/ocireg/blobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (

"ocm.software/ocm/api/oci/cpi"
"ocm.software/ocm/api/oci/extensions/attrs/cacheattr"
"ocm.software/ocm/api/tech/docker/resolve"
"ocm.software/ocm/api/tech/oras"
"ocm.software/ocm/api/utils/accessio"
"ocm.software/ocm/api/utils/blobaccess/blobaccess"
)
Expand All @@ -23,20 +23,20 @@ type BlobContainer interface {

type blobContainer struct {
accessio.StaticAllocatable
fetcher resolve.Fetcher
pusher resolve.Pusher
fetcher oras.Fetcher
pusher oras.Pusher
mime string
}

type BlobContainers struct {
lock sync.Mutex
cache accessio.BlobCache
fetcher resolve.Fetcher
pusher resolve.Pusher
fetcher oras.Fetcher
pusher oras.Pusher
mimes map[string]BlobContainer
}

func NewBlobContainers(ctx cpi.Context, fetcher remotes.Fetcher, pusher resolve.Pusher) *BlobContainers {
func NewBlobContainers(ctx cpi.Context, fetcher remotes.Fetcher, pusher oras.Pusher) *BlobContainers {
return &BlobContainers{
cache: cacheattr.Get(ctx),
fetcher: fetcher,
Expand Down Expand Up @@ -73,15 +73,15 @@ func (c *BlobContainers) Release() error {
return list.Result()
}

func newBlobContainer(mime string, fetcher resolve.Fetcher, pusher resolve.Pusher) *blobContainer {
func newBlobContainer(mime string, fetcher oras.Fetcher, pusher oras.Pusher) *blobContainer {
return &blobContainer{
mime: mime,
fetcher: fetcher,
pusher: pusher,
}
}

func NewBlobContainer(cache accessio.BlobCache, mime string, fetcher resolve.Fetcher, pusher resolve.Pusher) (BlobContainer, error) {
func NewBlobContainer(cache accessio.BlobCache, mime string, fetcher oras.Fetcher, pusher oras.Pusher) (BlobContainer, error) {
c := newBlobContainer(mime, fetcher, pusher)

if cache == nil {
Expand Down
12 changes: 6 additions & 6 deletions api/oci/extensions/repositories/ocireg/namespace.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"ocm.software/ocm/api/oci/cpi"
"ocm.software/ocm/api/oci/cpi/support"
"ocm.software/ocm/api/oci/extensions/actions/oci-repository-prepare"
"ocm.software/ocm/api/tech/docker/resolve"
"ocm.software/ocm/api/tech/oras"
"ocm.software/ocm/api/utils/accessio"
"ocm.software/ocm/api/utils/blobaccess/blobaccess"
"ocm.software/ocm/api/utils/logging"
Expand All @@ -22,10 +22,10 @@ import (
type NamespaceContainer struct {
impl support.NamespaceAccessImpl
repo *RepositoryImpl
resolver resolve.Resolver
lister resolve.Lister
fetcher resolve.Fetcher
pusher resolve.Pusher
resolver oras.Resolver
lister oras.Lister
fetcher oras.Fetcher
pusher oras.Pusher
blobs *BlobContainers
checked bool
}
Expand Down Expand Up @@ -69,7 +69,7 @@ func (n *NamespaceContainer) SetImplementation(impl support.NamespaceAccessImpl)
n.impl = impl
}

func (n *NamespaceContainer) getPusher(vers string) (resolve.Pusher, error) {
func (n *NamespaceContainer) getPusher(vers string) (oras.Pusher, error) {
err := n.assureCreated()
if err != nil {
return nil, err
Expand Down
92 changes: 44 additions & 48 deletions api/oci/extensions/repositories/ocireg/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,18 @@ import (
"path"
"strings"

"github.com/containerd/containerd/remotes/docker/config"
"github.com/containerd/errdefs"
"github.com/mandelsoft/goutils/errors"
"github.com/mandelsoft/logging"
"oras.land/oras-go/v2/registry/remote/auth"
"oras.land/oras-go/v2/registry/remote/retry"

"ocm.software/ocm/api/credentials"
"ocm.software/ocm/api/datacontext/attrs/rootcertsattr"
"ocm.software/ocm/api/oci/artdesc"
"ocm.software/ocm/api/oci/cpi"
"ocm.software/ocm/api/tech/docker"
"ocm.software/ocm/api/tech/docker/resolve"
"ocm.software/ocm/api/tech/oci/identity"
"ocm.software/ocm/api/tech/oras"
"ocm.software/ocm/api/utils"
ocmlog "ocm.software/ocm/api/utils/logging"
"ocm.software/ocm/api/utils/refmgmt"
Expand Down Expand Up @@ -114,7 +114,7 @@ func (r *RepositoryImpl) getCreds(comp string) (credentials.Credentials, error)
return identity.GetCredentials(r.GetContext(), r.info.Locator, comp)
}

func (r *RepositoryImpl) getResolver(comp string) (resolve.Resolver, error) {
func (r *RepositoryImpl) getResolver(comp string) (oras.Resolver, error) {
creds, err := r.getCreds(comp)
if err != nil {
if !errors.IsErrUnknownKind(err, credentials.KIND_CONSUMER) {
Expand All @@ -126,57 +126,53 @@ func (r *RepositoryImpl) getResolver(comp string) (resolve.Resolver, error) {
logger.Trace("no credentials")
}

opts := docker.ResolverOptions{
Hosts: docker.ConvertHosts(config.ConfigureHosts(context.Background(), config.HostOptions{
UpdateClient: func(client *http.Client) error {
// copy from http.DefaultTransport with a roundtripper injection
client.Transport = ocmlog.NewRoundTripper(client.Transport, logger)
return nil
},
Credentials: func(host string) (string, string, error) {
authCreds := auth.Credential{}
if creds != nil {
pass := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN)
if pass == "" {
pass = creds.GetProperty(credentials.ATTR_PASSWORD)
}
authCreds.Username = creds.GetProperty(credentials.ATTR_USERNAME)
authCreds.Password = pass
}
jakobmoellerdev marked this conversation as resolved.
Show resolved Hide resolved

client := retry.DefaultClient
client.Transport = ocmlog.NewRoundTripper(retry.DefaultClient.Transport, logger)
if r.info.Scheme == "https" {
// set up TLS
//nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version.
conf := &tls.Config{
// MinVersion: tls.VersionTLS13,
RootCAs: func() *x509.CertPool {
var rootCAs *x509.CertPool
if creds != nil {
p := creds.GetProperty(credentials.ATTR_IDENTITY_TOKEN)
if p == "" {
p = creds.GetProperty(credentials.ATTR_PASSWORD)
}
pw := ""
if p != "" {
pw = "***"
c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY)
if c != "" {
rootCAs = x509.NewCertPool()
rootCAs.AppendCertsFromPEM([]byte(c))
}
logger.Trace("query credentials", ocmlog.ATTR_USER, creds.GetProperty(credentials.ATTR_USERNAME), "pass", pw)
return creds.GetProperty(credentials.ATTR_USERNAME), p, nil
}
logger.Trace("no credentials")
return "", "", nil
},
DefaultScheme: r.info.Scheme,
//nolint:gosec // used like the default, there are OCI servers (quay.io) not working with min version.
DefaultTLS: func() *tls.Config {
if r.info.Scheme == "http" {
return nil
}
return &tls.Config{
// MinVersion: tls.VersionTLS13,
RootCAs: func() *x509.CertPool {
var rootCAs *x509.CertPool
if creds != nil {
c := creds.GetProperty(credentials.ATTR_CERTIFICATE_AUTHORITY)
if c != "" {
rootCAs = x509.NewCertPool()
rootCAs.AppendCertsFromPEM([]byte(c))
}
}
if rootCAs == nil {
rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true)
}
return rootCAs
}(),
if rootCAs == nil {
rootCAs = rootcertsattr.Get(r.GetContext()).GetRootCertPool(true)
}
return rootCAs
}(),
})),
}
client.Transport = ocmlog.NewRoundTripper(retry.NewTransport(&http.Transport{
TLSClientConfig: conf,
}), logger)
}

authClient := &auth.Client{
Client: client,
Cache: auth.NewCache(),
Credential: auth.StaticCredential(r.info.HostPort(), authCreds),
}

return docker.NewResolver(opts), nil
return oras.New(oras.ClientOptions{
Client: authClient,
PlainHTTP: r.info.Scheme == "http",
}), nil
}

func (r *RepositoryImpl) GetRef(comp, vers string) string {
Expand Down
13 changes: 7 additions & 6 deletions api/oci/extensions/repositories/ocireg/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (

"ocm.software/ocm/api/oci/artdesc"
"ocm.software/ocm/api/oci/cpi"
"ocm.software/ocm/api/tech/docker/resolve"
"ocm.software/ocm/api/tech/oras"
"ocm.software/ocm/api/utils/accessio"
"ocm.software/ocm/api/utils/blobaccess/blobaccess"
"ocm.software/ocm/api/utils/logging"
Expand Down Expand Up @@ -81,28 +81,29 @@ func readAll(reader io.ReadCloser, err error) ([]byte, error) {
return data, nil
}

func push(ctx context.Context, p resolve.Pusher, blob blobaccess.BlobAccess) error {
func push(ctx context.Context, p oras.Pusher, blob blobaccess.BlobAccess) error {
desc := *artdesc.DefaultBlobDescriptor(blob)
return pushData(ctx, p, desc, blob)
}

func pushData(ctx context.Context, p resolve.Pusher, desc artdesc.Descriptor, data blobaccess.DataAccess) error {
func pushData(ctx context.Context, p oras.Pusher, desc artdesc.Descriptor, data blobaccess.DataAccess) error {
key := remotes.MakeRefKey(ctx, desc)
if desc.Size == 0 {
desc.Size = -1
}

logging.Logger().Debug("*** push blob", "mediatype", desc.MediaType, "digest", desc.Digest, "key", key)
req, err := p.Push(ctx, desc, data)
if err != nil {
if err := p.Push(ctx, desc, data); err != nil {
if errdefs.IsAlreadyExists(err) {
logging.Logger().Debug("blob already exists", "mediatype", desc.MediaType, "digest", desc.Digest)

return nil
}

return fmt.Errorf("failed to push: %w", err)
}
return req.Commit(ctx, desc.Size, desc.Digest)

return nil
}

var dummyContext = nologger()
Expand Down
4 changes: 0 additions & 4 deletions api/tech/docker/README.md

This file was deleted.

58 changes: 0 additions & 58 deletions api/tech/docker/errors/errors.go

This file was deleted.

Loading
Loading