Skip to content

client: add test for deleted blobs in remote cache #6273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 7, 2025
Merged

client: add test for deleted blobs in remote cache #6273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cache/remotecache/azblob/exporter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/containerd/containerd/v2/pkg/labels"
"github.com/moby/buildkit/cache/remotecache"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
Expand Down Expand Up @@ -110,7 +111,7 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
layerDone(nil)
}

la := &v1.LayerAnnotations{
la := &cacheimporttypes.LayerAnnotations{
DiffID: diffID,
Size: dgstPair.Descriptor.Size,
MediaType: dgstPair.Descriptor.MediaType,
Expand Down
5 changes: 3 additions & 2 deletions cache/remotecache/azblob/importer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/buildkit/cache/remotecache"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
Expand Down Expand Up @@ -118,7 +119,7 @@ func (ci *importer) loadManifest(ctx context.Context, name string) (*v1.CacheCha

bklog.G(ctx).Debugf("imported config: %s", string(bytes))

var config v1.CacheConfig
var config cacheimporttypes.CacheConfig
if err := json.Unmarshal(bytes, &config); err != nil {
return nil, errors.WithStack(err)
}
Expand All @@ -142,7 +143,7 @@ func (ci *importer) loadManifest(ctx context.Context, name string) (*v1.CacheCha
return cc, nil
}

func (ci *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorProviderPair, error) {
func (ci *importer) makeDescriptorProviderPair(l cacheimporttypes.CacheLayer) (*v1.DescriptorProviderPair, error) {
if l.Annotations == nil {
return nil, errors.Errorf("cache layer with missing annotations")
}
Expand Down
3 changes: 2 additions & 1 deletion cache/remotecache/export.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/core/images"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
Expand Down Expand Up @@ -219,7 +220,7 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string
desc := ocispecs.Descriptor{
Digest: dgst,
Size: int64(len(dt)),
MediaType: v1.CacheConfigMediaTypeV0,
MediaType: cacheimporttypes.CacheConfigMediaTypeV0,
}
configDone := progress.OneOff(ctx, fmt.Sprintf("writing config %s", dgst))
if err := content.WriteBlob(ctx, ce.ingester, dgst.String(), bytes.NewReader(dt), desc); err != nil {
Expand Down
7 changes: 4 additions & 3 deletions cache/remotecache/gha/gha.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/buildkit/cache/remotecache"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
Expand Down Expand Up @@ -267,7 +268,7 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
}
layerDone(nil)
}
la := &v1.LayerAnnotations{
la := &cacheimporttypes.LayerAnnotations{
DiffID: diffID,
Size: dgstPair.Descriptor.Size,
MediaType: dgstPair.Descriptor.MediaType,
Expand Down Expand Up @@ -328,7 +329,7 @@ func NewImporter(c *Config) (remotecache.Importer, error) {
return &importer{cache: cache, config: c}, nil
}

func (ci *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorProviderPair, error) {
func (ci *importer) makeDescriptorProviderPair(l cacheimporttypes.CacheLayer) (*v1.DescriptorProviderPair, error) {
if l.Annotations == nil {
return nil, errors.Errorf("cache layer with missing annotations")
}
Expand Down Expand Up @@ -376,7 +377,7 @@ func (ci *importer) loadScope(ctx context.Context, scope string) (*v1.CacheChain
return nil, err
}

var config v1.CacheConfig
var config cacheimporttypes.CacheConfig
if err := json.Unmarshal(buf.Bytes(), &config); err != nil {
return nil, errors.WithStack(err)
}
Expand Down
9 changes: 5 additions & 4 deletions cache/remotecache/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/containerd/containerd/v2/core/images"
"github.com/containerd/containerd/v2/pkg/labels"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
Expand Down Expand Up @@ -69,7 +70,7 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
}

for _, m := range mfst.Manifests {
if m.MediaType == v1.CacheConfigMediaTypeV0 {
if m.MediaType == cacheimporttypes.CacheConfigMediaTypeV0 {
configDesc = m
continue
}
Expand All @@ -84,7 +85,7 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
return nil, err
}

if mfst.Config.MediaType == v1.CacheConfigMediaTypeV0 {
if mfst.Config.MediaType == cacheimporttypes.CacheConfigMediaTypeV0 {
configDesc = mfst.Config
}
for _, m := range mfst.Layers {
Expand Down Expand Up @@ -201,7 +202,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
return nil
}

var config v1.CacheConfig
var config cacheimporttypes.CacheConfig
if err := json.Unmarshal(img.Cache, &config.Records); err != nil {
return errors.WithStack(err)
}
Expand All @@ -227,7 +228,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
Descriptor: m,
Provider: ci.provider,
}
config.Layers = append(config.Layers, v1.CacheLayer{
config.Layers = append(config.Layers, cacheimporttypes.CacheLayer{
Blob: m.Digest,
ParentIndex: i - 1,
})
Expand Down
11 changes: 6 additions & 5 deletions cache/remotecache/inline/inline.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/containerd/containerd/v2/pkg/labels"
"github.com/moby/buildkit/cache/remotecache"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
Expand Down Expand Up @@ -122,21 +123,21 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest)
} else {
// The layers of the result are not in the same order as the image, so we
// have to use ChainedResult to specify each layer of the result individually.
chainedResult := v1.ChainedResult{}
chainedResult := cacheimporttypes.ChainedResult{}
for _, resultBlob := range resultBlobs {
idx, ok := blobIndexes[resultBlob]
if !ok {
return nil, errors.Errorf("failed to find blob %s in layers", resultBlob)
}
chainedResult.LayerIndexes = append(chainedResult.LayerIndexes, idx)
}
r.Results[j] = v1.CacheResult{}
r.Results[j] = cacheimporttypes.CacheResult{}
r.ChainedResults = append(r.ChainedResults, chainedResult)
}
// remove any CacheResults that had to be converted to the ChainedResult format.
var filteredResults []v1.CacheResult
var filteredResults []cacheimporttypes.CacheResult
for _, rr := range r.Results {
if rr != (v1.CacheResult{}) {
if rr != (cacheimporttypes.CacheResult{}) {
filteredResults = append(filteredResults, rr)
}
}
Expand All @@ -154,7 +155,7 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest)
return dt, nil
}

func layerToBlobs(idx int, layers []v1.CacheLayer) []digest.Digest {
func layerToBlobs(idx int, layers []cacheimporttypes.CacheLayer) []digest.Digest {
var ds []digest.Digest
for idx != -1 {
layer := layers[idx]
Expand Down
9 changes: 5 additions & 4 deletions cache/remotecache/s3/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/containerd/containerd/v2/pkg/labels"
"github.com/moby/buildkit/cache/remotecache"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/compression"
Expand Down Expand Up @@ -261,7 +262,7 @@ func (e *exporter) Finalize(ctx context.Context) (map[string]string, error) {
layerDone(nil)
}

la := &v1.LayerAnnotations{
la := &cacheimporttypes.LayerAnnotations{
DiffID: diffID,
Size: dgstPair.Descriptor.Size,
MediaType: dgstPair.Descriptor.MediaType,
Expand Down Expand Up @@ -316,7 +317,7 @@ type importer struct {
config Config
}

func (i *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorProviderPair, error) {
func (i *importer) makeDescriptorProviderPair(l cacheimporttypes.CacheLayer) (*v1.DescriptorProviderPair, error) {
if l.Annotations == nil {
return nil, errors.Errorf("cache layer with missing annotations")
}
Expand Down Expand Up @@ -344,7 +345,7 @@ func (i *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorPr
}

func (i *importer) load(ctx context.Context) (*v1.CacheChains, error) {
var config v1.CacheConfig
var config cacheimporttypes.CacheConfig
found, err := i.s3Client.getManifest(ctx, i.s3Client.manifestKey(i.config.Names[0]), &config)
if err != nil {
return nil, err
Expand Down Expand Up @@ -427,7 +428,7 @@ func newS3Client(ctx context.Context, config Config) (*s3Client, error) {
}, nil
}

func (s3Client *s3Client) getManifest(ctx context.Context, key string, config *v1.CacheConfig) (bool, error) {
func (s3Client *s3Client) getManifest(ctx context.Context, key string, config *cacheimporttypes.CacheConfig) (bool, error) {
input := &s3.GetObjectInput{
Bucket: &s3Client.bucket,
Key: &key,
Expand Down
5 changes: 3 additions & 2 deletions cache/remotecache/v1/chains.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/cespare/xxhash/v2"
"github.com/containerd/containerd/v2/core/content"
cerrdefs "github.com/containerd/errdefs"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
digest "github.com/opencontainers/go-digest"
Expand Down Expand Up @@ -208,7 +209,7 @@ func IntersectAll[T comparable](
// Marshal aims to validate, normalize and sort the output to ensure a
// consistent digest (since cache configs are typically uploaded and stored in
// content-addressable OCI registries).
func (c *CacheChains) Marshal(ctx context.Context) (*CacheConfig, DescriptorProvider, error) {
func (c *CacheChains) Marshal(ctx context.Context) (*cacheimporttypes.CacheConfig, DescriptorProvider, error) {
st := &marshalState{
chainsByID: map[string]int{},
descriptors: DescriptorProvider{},
Expand All @@ -221,7 +222,7 @@ func (c *CacheChains) Marshal(ctx context.Context) (*CacheConfig, DescriptorProv
}
}

cc := CacheConfig{
cc := cacheimporttypes.CacheConfig{
Layers: st.layers,
Records: st.records,
}
Expand Down
9 changes: 5 additions & 4 deletions cache/remotecache/v1/parse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,23 @@ package cacheimport
import (
"encoding/json"

cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/contentutil"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

func Parse(configJSON []byte, provider DescriptorProvider, t solver.CacheExporterTarget) error {
var config CacheConfig
var config cacheimporttypes.CacheConfig
if err := json.Unmarshal(configJSON, &config); err != nil {
return errors.WithStack(err)
}

return ParseConfig(config, provider, t)
}

func ParseConfig(config CacheConfig, provider DescriptorProvider, t solver.CacheExporterTarget) error {
func ParseConfig(config cacheimporttypes.CacheConfig, provider DescriptorProvider, t solver.CacheExporterTarget) error {
cache := map[int]solver.CacheExporterRecord{}

for i := range config.Records {
Expand All @@ -29,7 +30,7 @@ func ParseConfig(config CacheConfig, provider DescriptorProvider, t solver.Cache
return nil
}

func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver.CacheExporterTarget, cache map[int]solver.CacheExporterRecord) (solver.CacheExporterRecord, error) {
func parseRecord(cc cacheimporttypes.CacheConfig, idx int, provider DescriptorProvider, t solver.CacheExporterTarget, cache map[int]solver.CacheExporterRecord) (solver.CacheExporterRecord, error) {
if r, ok := cache[idx]; ok {
if r == nil {
return nil, errors.Errorf("invalid looping record")
Expand Down Expand Up @@ -112,7 +113,7 @@ func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver.
return r, nil
}

func getRemoteChain(layers []CacheLayer, idx int, provider DescriptorProvider, visited map[int]struct{}) (*solver.Remote, error) {
func getRemoteChain(layers []cacheimporttypes.CacheLayer, idx int, provider DescriptorProvider, visited map[int]struct{}) (*solver.Remote, error) {
if _, ok := visited[idx]; ok {
return nil, errors.Errorf("invalid looping layer")
}
Expand Down
2 changes: 1 addition & 1 deletion cache/remotecache/v1/doc.go → cache/remotecache/v1/types/doc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package cacheimport
package cacheimporttypes

// Distributable build cache
//
Expand Down
2 changes: 1 addition & 1 deletion cache/remotecache/v1/spec.go → cache/remotecache/v1/types/spec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package cacheimport
package cacheimporttypes

import (
"time"
Expand Down
Loading