Skip to content

Commit 7fbc646

Browse files
tonistiigicrazy-max
tonistiigi
authored and
crazy-max
committed
client: add test for deleted blobs in remote cache
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
1 parent 3b5fb90 commit 7fbc646

File tree

14 files changed

+197
-43
lines changed

14 files changed

+197
-43
lines changed

‎cache/remotecache/azblob/exporter.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/containerd/containerd/v2/pkg/labels"
1919
"github.com/moby/buildkit/cache/remotecache"
2020
v1 "github.com/moby/buildkit/cache/remotecache/v1"
21+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
2122
"github.com/moby/buildkit/session"
2223
"github.com/moby/buildkit/solver"
2324
"github.com/moby/buildkit/util/bklog"
@@ -110,7 +111,7 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
110111
layerDone(nil)
111112
}
112113

113-
la := &v1.LayerAnnotations{
114+
la := &cacheimporttypes.LayerAnnotations{
114115
DiffID: diffID,
115116
Size: dgstPair.Descriptor.Size,
116117
MediaType: dgstPair.Descriptor.MediaType,

‎cache/remotecache/azblob/importer.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
cerrdefs "github.com/containerd/errdefs"
1515
"github.com/moby/buildkit/cache/remotecache"
1616
v1 "github.com/moby/buildkit/cache/remotecache/v1"
17+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
1718
"github.com/moby/buildkit/session"
1819
"github.com/moby/buildkit/solver"
1920
"github.com/moby/buildkit/util/bklog"
@@ -118,7 +119,7 @@ func (ci *importer) loadManifest(ctx context.Context, name string) (*v1.CacheCha
118119

119120
bklog.G(ctx).Debugf("imported config: %s", string(bytes))
120121

121-
var config v1.CacheConfig
122+
var config cacheimporttypes.CacheConfig
122123
if err := json.Unmarshal(bytes, &config); err != nil {
123124
return nil, errors.WithStack(err)
124125
}
@@ -142,7 +143,7 @@ func (ci *importer) loadManifest(ctx context.Context, name string) (*v1.CacheCha
142143
return cc, nil
143144
}
144145

145-
func (ci *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorProviderPair, error) {
146+
func (ci *importer) makeDescriptorProviderPair(l cacheimporttypes.CacheLayer) (*v1.DescriptorProviderPair, error) {
146147
if l.Annotations == nil {
147148
return nil, errors.Errorf("cache layer with missing annotations")
148149
}

‎cache/remotecache/export.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/containerd/containerd/v2/core/content"
1010
"github.com/containerd/containerd/v2/core/images"
1111
v1 "github.com/moby/buildkit/cache/remotecache/v1"
12+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
1213
"github.com/moby/buildkit/session"
1314
"github.com/moby/buildkit/solver"
1415
"github.com/moby/buildkit/util/bklog"
@@ -219,7 +220,7 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string
219220
desc := ocispecs.Descriptor{
220221
Digest: dgst,
221222
Size: int64(len(dt)),
222-
MediaType: v1.CacheConfigMediaTypeV0,
223+
MediaType: cacheimporttypes.CacheConfigMediaTypeV0,
223224
}
224225
configDone := progress.OneOff(ctx, fmt.Sprintf("writing config %s", dgst))
225226
if err := content.WriteBlob(ctx, ce.ingester, dgst.String(), bytes.NewReader(dt), desc); err != nil {

‎cache/remotecache/gha/gha.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
cerrdefs "github.com/containerd/errdefs"
1818
"github.com/moby/buildkit/cache/remotecache"
1919
v1 "github.com/moby/buildkit/cache/remotecache/v1"
20+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
2021
"github.com/moby/buildkit/session"
2122
"github.com/moby/buildkit/solver"
2223
"github.com/moby/buildkit/util/bklog"
@@ -267,7 +268,7 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
267268
}
268269
layerDone(nil)
269270
}
270-
la := &v1.LayerAnnotations{
271+
la := &cacheimporttypes.LayerAnnotations{
271272
DiffID: diffID,
272273
Size: dgstPair.Descriptor.Size,
273274
MediaType: dgstPair.Descriptor.MediaType,
@@ -328,7 +329,7 @@ func NewImporter(c *Config) (remotecache.Importer, error) {
328329
return &importer{cache: cache, config: c}, nil
329330
}
330331

331-
func (ci *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorProviderPair, error) {
332+
func (ci *importer) makeDescriptorProviderPair(l cacheimporttypes.CacheLayer) (*v1.DescriptorProviderPair, error) {
332333
if l.Annotations == nil {
333334
return nil, errors.Errorf("cache layer with missing annotations")
334335
}
@@ -376,7 +377,7 @@ func (ci *importer) loadScope(ctx context.Context, scope string) (*v1.CacheChain
376377
return nil, err
377378
}
378379

379-
var config v1.CacheConfig
380+
var config cacheimporttypes.CacheConfig
380381
if err := json.Unmarshal(buf.Bytes(), &config); err != nil {
381382
return nil, errors.WithStack(err)
382383
}

‎cache/remotecache/import.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/containerd/containerd/v2/core/images"
1313
"github.com/containerd/containerd/v2/pkg/labels"
1414
v1 "github.com/moby/buildkit/cache/remotecache/v1"
15+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
1516
"github.com/moby/buildkit/session"
1617
"github.com/moby/buildkit/solver"
1718
"github.com/moby/buildkit/util/bklog"
@@ -69,7 +70,7 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
6970
}
7071

7172
for _, m := range mfst.Manifests {
72-
if m.MediaType == v1.CacheConfigMediaTypeV0 {
73+
if m.MediaType == cacheimporttypes.CacheConfigMediaTypeV0 {
7374
configDesc = m
7475
continue
7576
}
@@ -84,7 +85,7 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
8485
return nil, err
8586
}
8687

87-
if mfst.Config.MediaType == v1.CacheConfigMediaTypeV0 {
88+
if mfst.Config.MediaType == cacheimporttypes.CacheConfigMediaTypeV0 {
8889
configDesc = mfst.Config
8990
}
9091
for _, m := range mfst.Layers {
@@ -201,7 +202,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
201202
return nil
202203
}
203204

204-
var config v1.CacheConfig
205+
var config cacheimporttypes.CacheConfig
205206
if err := json.Unmarshal(img.Cache, &config.Records); err != nil {
206207
return errors.WithStack(err)
207208
}
@@ -227,7 +228,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
227228
Descriptor: m,
228229
Provider: ci.provider,
229230
}
230-
config.Layers = append(config.Layers, v1.CacheLayer{
231+
config.Layers = append(config.Layers, cacheimporttypes.CacheLayer{
231232
Blob: m.Digest,
232233
ParentIndex: i - 1,
233234
})

‎cache/remotecache/inline/inline.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/containerd/containerd/v2/pkg/labels"
99
"github.com/moby/buildkit/cache/remotecache"
1010
v1 "github.com/moby/buildkit/cache/remotecache/v1"
11+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
1112
"github.com/moby/buildkit/session"
1213
"github.com/moby/buildkit/solver"
1314
"github.com/moby/buildkit/util/bklog"
@@ -122,21 +123,21 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest)
122123
} else {
123124
// The layers of the result are not in the same order as the image, so we
124125
// have to use ChainedResult to specify each layer of the result individually.
125-
chainedResult := v1.ChainedResult{}
126+
chainedResult := cacheimporttypes.ChainedResult{}
126127
for _, resultBlob := range resultBlobs {
127128
idx, ok := blobIndexes[resultBlob]
128129
if !ok {
129130
return nil, errors.Errorf("failed to find blob %s in layers", resultBlob)
130131
}
131132
chainedResult.LayerIndexes = append(chainedResult.LayerIndexes, idx)
132133
}
133-
r.Results[j] = v1.CacheResult{}
134+
r.Results[j] = cacheimporttypes.CacheResult{}
134135
r.ChainedResults = append(r.ChainedResults, chainedResult)
135136
}
136137
// remove any CacheResults that had to be converted to the ChainedResult format.
137-
var filteredResults []v1.CacheResult
138+
var filteredResults []cacheimporttypes.CacheResult
138139
for _, rr := range r.Results {
139-
if rr != (v1.CacheResult{}) {
140+
if rr != (cacheimporttypes.CacheResult{}) {
140141
filteredResults = append(filteredResults, rr)
141142
}
142143
}
@@ -154,7 +155,7 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest)
154155
return dt, nil
155156
}
156157

157-
func layerToBlobs(idx int, layers []v1.CacheLayer) []digest.Digest {
158+
func layerToBlobs(idx int, layers []cacheimporttypes.CacheLayer) []digest.Digest {
158159
var ds []digest.Digest
159160
for idx != -1 {
160161
layer := layers[idx]

‎cache/remotecache/s3/s3.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/containerd/containerd/v2/pkg/labels"
2222
"github.com/moby/buildkit/cache/remotecache"
2323
v1 "github.com/moby/buildkit/cache/remotecache/v1"
24+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
2425
"github.com/moby/buildkit/session"
2526
"github.com/moby/buildkit/solver"
2627
"github.com/moby/buildkit/util/compression"
@@ -261,7 +262,7 @@ func (e *exporter) Finalize(ctx context.Context) (map[string]string, error) {
261262
layerDone(nil)
262263
}
263264

264-
la := &v1.LayerAnnotations{
265+
la := &cacheimporttypes.LayerAnnotations{
265266
DiffID: diffID,
266267
Size: dgstPair.Descriptor.Size,
267268
MediaType: dgstPair.Descriptor.MediaType,
@@ -316,7 +317,7 @@ type importer struct {
316317
config Config
317318
}
318319

319-
func (i *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorProviderPair, error) {
320+
func (i *importer) makeDescriptorProviderPair(l cacheimporttypes.CacheLayer) (*v1.DescriptorProviderPair, error) {
320321
if l.Annotations == nil {
321322
return nil, errors.Errorf("cache layer with missing annotations")
322323
}
@@ -344,7 +345,7 @@ func (i *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorPr
344345
}
345346

346347
func (i *importer) load(ctx context.Context) (*v1.CacheChains, error) {
347-
var config v1.CacheConfig
348+
var config cacheimporttypes.CacheConfig
348349
found, err := i.s3Client.getManifest(ctx, i.s3Client.manifestKey(i.config.Names[0]), &config)
349350
if err != nil {
350351
return nil, err
@@ -427,7 +428,7 @@ func newS3Client(ctx context.Context, config Config) (*s3Client, error) {
427428
}, nil
428429
}
429430

430-
func (s3Client *s3Client) getManifest(ctx context.Context, key string, config *v1.CacheConfig) (bool, error) {
431+
func (s3Client *s3Client) getManifest(ctx context.Context, key string, config *cacheimporttypes.CacheConfig) (bool, error) {
431432
input := &s3.GetObjectInput{
432433
Bucket: &s3Client.bucket,
433434
Key: &key,

‎cache/remotecache/v1/chains.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/cespare/xxhash/v2"
1212
"github.com/containerd/containerd/v2/core/content"
1313
cerrdefs "github.com/containerd/errdefs"
14+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
1415
"github.com/moby/buildkit/session"
1516
"github.com/moby/buildkit/solver"
1617
digest "github.com/opencontainers/go-digest"
@@ -208,7 +209,7 @@ func IntersectAll[T comparable](
208209
// Marshal aims to validate, normalize and sort the output to ensure a
209210
// consistent digest (since cache configs are typically uploaded and stored in
210211
// content-addressable OCI registries).
211-
func (c *CacheChains) Marshal(ctx context.Context) (*CacheConfig, DescriptorProvider, error) {
212+
func (c *CacheChains) Marshal(ctx context.Context) (*cacheimporttypes.CacheConfig, DescriptorProvider, error) {
212213
st := &marshalState{
213214
chainsByID: map[string]int{},
214215
descriptors: DescriptorProvider{},
@@ -221,7 +222,7 @@ func (c *CacheChains) Marshal(ctx context.Context) (*CacheConfig, DescriptorProv
221222
}
222223
}
223224

224-
cc := CacheConfig{
225+
cc := cacheimporttypes.CacheConfig{
225226
Layers: st.layers,
226227
Records: st.records,
227228
}

‎cache/remotecache/v1/parse.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,23 @@ package cacheimport
33
import (
44
"encoding/json"
55

6+
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
67
"github.com/moby/buildkit/solver"
78
"github.com/moby/buildkit/util/contentutil"
89
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
910
"github.com/pkg/errors"
1011
)
1112

1213
func Parse(configJSON []byte, provider DescriptorProvider, t solver.CacheExporterTarget) error {
13-
var config CacheConfig
14+
var config cacheimporttypes.CacheConfig
1415
if err := json.Unmarshal(configJSON, &config); err != nil {
1516
return errors.WithStack(err)
1617
}
1718

1819
return ParseConfig(config, provider, t)
1920
}
2021

21-
func ParseConfig(config CacheConfig, provider DescriptorProvider, t solver.CacheExporterTarget) error {
22+
func ParseConfig(config cacheimporttypes.CacheConfig, provider DescriptorProvider, t solver.CacheExporterTarget) error {
2223
cache := map[int]solver.CacheExporterRecord{}
2324

2425
for i := range config.Records {
@@ -29,7 +30,7 @@ func ParseConfig(config CacheConfig, provider DescriptorProvider, t solver.Cache
2930
return nil
3031
}
3132

32-
func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver.CacheExporterTarget, cache map[int]solver.CacheExporterRecord) (solver.CacheExporterRecord, error) {
33+
func parseRecord(cc cacheimporttypes.CacheConfig, idx int, provider DescriptorProvider, t solver.CacheExporterTarget, cache map[int]solver.CacheExporterRecord) (solver.CacheExporterRecord, error) {
3334
if r, ok := cache[idx]; ok {
3435
if r == nil {
3536
return nil, errors.Errorf("invalid looping record")
@@ -112,7 +113,7 @@ func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver.
112113
return r, nil
113114
}
114115

115-
func getRemoteChain(layers []CacheLayer, idx int, provider DescriptorProvider, visited map[int]struct{}) (*solver.Remote, error) {
116+
func getRemoteChain(layers []cacheimporttypes.CacheLayer, idx int, provider DescriptorProvider, visited map[int]struct{}) (*solver.Remote, error) {
116117
if _, ok := visited[idx]; ok {
117118
return nil, errors.Errorf("invalid looping layer")
118119
}

‎cache/remotecache/v1/doc.go‎ renamed to ‎cache/remotecache/v1/types/doc.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package cacheimport
1+
package cacheimporttypes
22

33
// Distributable build cache
44
//

0 commit comments

Comments
 (0)