Skip to content

Commit

Permalink
feat: add new type Store in content
Browse files Browse the repository at this point in the history
Store wrap the content store, we can use Store to complete custom feature.
Such as call the UpdateTime in content store ReaderAt.
We also move the newStore and NewDB to newContent.

Signed-off-by: Yadong Ding <ding_yadong@foxmail.com>
  • Loading branch information
Desiki-high committed Jul 13, 2023
1 parent c893d36 commit 6a1b79b
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 35 deletions.
22 changes: 19 additions & 3 deletions pkg/content/content.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ import (
"context"
"encoding/binary"
"fmt"
"path/filepath"
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/metadata"
"github.com/dustin/go-humanize"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
bolt "go.etcd.io/bbolt"
)
Expand All @@ -42,15 +44,29 @@ type Content struct {
threshold int64
}

func NewContent(db *metadata.DB, threshold string) (*Content, error) {
func NewContent(contentDir string, databaseDir string, threshold string) (*Content, error) {
store, err := newStore(contentDir)
if err != nil {
return nil, errors.Wrap(err, "create local provider content store")
}
bdb, err := bolt.Open(filepath.Join(databaseDir, "meta.db"), 0655, nil)
if err != nil {
return nil, errors.Wrap(err, "create local provider database")
}
db := metadata.NewDB(bdb, store, nil)
if err := db.Init(context.Background()); err != nil {
return nil, err
}
t, err := humanize.ParseBytes(threshold)
if err != nil {
return nil, err
}
return &Content{
content := Content{
db: db,
threshold: int64(t),
}, nil
}
store.Init(&content)
return &content, nil
}

// return the content store in db
Expand Down
12 changes: 1 addition & 11 deletions pkg/content/content_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,16 @@
package content

import (
"context"
"os"
"testing"

"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/metadata"
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
)

func TestContenSize(t *testing.T) {
os.MkdirAll("./tmp", 0755)
defer os.RemoveAll("./tmp")
store, err := local.NewStore("./tmp")
require.NoError(t, err)
bdb, err := bolt.Open("./tmp/metadata.db", 0655, nil)
require.NoError(t, err)
db := metadata.NewDB(bdb, store, nil)
require.NoError(t, db.Init(context.Background()))
content, err := NewContent(db, "1000MB")
content, err := NewContent("./tmp", "./tmp", "1000MB")
require.NoError(t, err)
size, err := content.Size()
require.NoError(t, err)
Expand Down
19 changes: 2 additions & 17 deletions pkg/content/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,12 @@ import (

"github.com/containerd/containerd"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/goharbor/acceleration-service/pkg/remote"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)

type LocalProvider struct {
Expand All @@ -52,19 +49,7 @@ func NewLocalProvider(
if err := os.MkdirAll(contentDir, 0755); err != nil {
return nil, nil, errors.Wrap(err, "create local provider work directory")
}
store, err := local.NewLabeledStore(contentDir, newMemoryLabelStore())
if err != nil {
return nil, nil, errors.Wrap(err, "create local provider content store")
}
bdb, err := bolt.Open(filepath.Join(workDir, "meta.db"), 0655, nil)
if err != nil {
return nil, nil, errors.Wrap(err, "create local provider database")
}
db := metadata.NewDB(bdb, store, nil)
if err := db.Init(context.Background()); err != nil {
return nil, nil, err
}
content, err := NewContent(db, threshold)
content, err := NewContent(contentDir, workDir, threshold)
if err != nil {
return nil, nil, errors.Wrap(err, "create local provider content")
}
Expand Down Expand Up @@ -100,7 +85,7 @@ func (pvd *LocalProvider) Pull(ctx context.Context, ref string) error {
PlatformMatcher: pvd.platformMC,
}

img, err := fetch(ctx, pvd.ContentStore(), rc, ref, 0, pvd.content)
img, err := fetch(ctx, pvd.ContentStore(), rc, ref, 0)
if err != nil {
return errors.Wrap(err, "pull source image")
}
Expand Down
8 changes: 4 additions & 4 deletions pkg/content/ported.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ var fetchSingleflight = &singleflight.Group{}

// Ported from containerd project, copyright The containerd Authors.
// github.com/containerd/containerd/blob/main/pull.go
func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteContext, ref string, limit int, content *Content) (images.Image, error) {
func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteContext, ref string, limit int) (images.Image, error) {
name, desc, err := rCtx.Resolver.Resolve(ctx, ref)
if err != nil {
return images.Image{}, fmt.Errorf("failed to resolve reference %q: %w", ref, err)
Expand Down Expand Up @@ -107,7 +107,7 @@ func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteCont
}

handlers := append(rCtx.BaseHandlers,
fetchHandler(store, fetcher, content),
fetchHandler(store, fetcher),
convertibleHandler,
childrenHandler,
appendDistSrcLabelHandler,
Expand Down Expand Up @@ -147,7 +147,7 @@ func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteCont

// Ported from containerd project, copyright The containerd Authors.
// https://github.com/containerd/containerd/blob/main/remotes/handlers.go
func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, content *Content) images.HandlerFunc {
func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher) images.HandlerFunc {
return func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
"digest": desc.Digest,
Expand All @@ -163,7 +163,7 @@ func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, content *C
return nil, remotes.Fetch(ctx, ingester, fetcher, desc)
})
if errdefs.IsAlreadyExists(err) {
return nil, content.UpdateTime(&desc.Digest)
return nil, nil
}
return nil, err
}
Expand Down
84 changes: 84 additions & 0 deletions pkg/content/store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package content

import (
"context"

containerdContent "github.com/containerd/containerd/content"
"github.com/containerd/containerd/content/local"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

// Store wrap the content store to complete custom feature
type Store struct {
// store is content store
store containerdContent.Store
// content is related to database
content *Content
}

// newStore returns a content store
func newStore(contentDir string) (*Store, error) {
store, err := local.NewLabeledStore(contentDir, newMemoryLabelStore())
return &Store{
store: store,
}, err
}

func (store *Store) Init(content *Content) {
store.content = content
}

func (store *Store) Info(ctx context.Context, dgst digest.Digest) (containerdContent.Info, error) {
return store.store.Info(ctx, dgst)
}

func (store *Store) Update(ctx context.Context, info containerdContent.Info, fieldpaths ...string) (containerdContent.Info, error) {
return store.store.Update(ctx, info, fieldpaths...)
}

func (store *Store) Walk(ctx context.Context, fn containerdContent.WalkFunc, filters ...string) error {
return store.store.Walk(ctx, fn, filters...)
}

func (store *Store) Delete(ctx context.Context, dgst digest.Digest) error {
return store.store.Delete(ctx, dgst)
}

func (store *Store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (containerdContent.ReaderAt, error) {
readerAt, err := store.store.ReaderAt(ctx, desc)
if err != nil {
return readerAt, err
}
return readerAt, store.content.UpdateTime(&desc.Digest)
}

func (store *Store) Status(ctx context.Context, ref string) (containerdContent.Status, error) {
return store.store.Status(ctx, ref)
}

func (store *Store) ListStatuses(ctx context.Context, filters ...string) ([]containerdContent.Status, error) {
return store.store.ListStatuses(ctx, filters...)
}

func (store *Store) Abort(ctx context.Context, ref string) error {
return store.store.Abort(ctx, ref)
}

func (store *Store) Writer(ctx context.Context, opts ...containerdContent.WriterOpt) (containerdContent.Writer, error) {
return store.store.Writer(ctx, opts...)
}

0 comments on commit 6a1b79b

Please sign in to comment.