From 962c287b1b651b05a5cb8070632ed4425eb7056d Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 28 Oct 2021 20:30:29 -0700 Subject: [PATCH] gha: fix handling removed blobs on reexport Signed-off-by: Tonis Tiigi --- cache/remotecache/export.go | 2 +- cache/remotecache/gha/gha.go | 2 +- cache/remotecache/inline/inline.go | 6 +++--- cache/remotecache/v1/chains.go | 5 +++-- cache/remotecache/v1/chains_test.go | 9 +++++---- cache/remotecache/v1/utils.go | 22 +++++++++++++++++----- solver/llbsolver/solver.go | 4 ++-- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/cache/remotecache/export.go b/cache/remotecache/export.go index 73ff2ac96b1c..1ca0c25d9e0e 100644 --- a/cache/remotecache/export.go +++ b/cache/remotecache/export.go @@ -68,7 +68,7 @@ func NewExporter(ingester content.Ingester, ref string, oci bool) Exporter { func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string, error) { res := make(map[string]string) - config, descs, err := ce.chains.Marshal() + config, descs, err := ce.chains.Marshal(ctx) if err != nil { return nil, err } diff --git a/cache/remotecache/gha/gha.go b/cache/remotecache/gha/gha.go index a8fded568f49..9458730b0695 100644 --- a/cache/remotecache/gha/gha.go +++ b/cache/remotecache/gha/gha.go @@ -106,7 +106,7 @@ func (ce *exporter) indexKey() string { func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) { // res := make(map[string]string) - config, descs, err := ce.chains.Marshal() + config, descs, err := ce.chains.Marshal(ctx) if err != nil { return nil, err } diff --git a/cache/remotecache/inline/inline.go b/cache/remotecache/inline/inline.go index 449c7c0fb51f..1d1c1180b32a 100644 --- a/cache/remotecache/inline/inline.go +++ b/cache/remotecache/inline/inline.go @@ -38,8 +38,8 @@ func (ce *exporter) reset() { ce.chains = cc } -func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) { - config, descs, err := ce.chains.Marshal() +func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest) ([]byte, error) { + config, descs, err := ce.chains.Marshal(ctx) if err != nil { return nil, err } @@ -63,7 +63,7 @@ func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) { return nil, err } - cfg, _, err := cc.Marshal() + cfg, _, err := cc.Marshal(ctx) if err != nil { return nil, err } diff --git a/cache/remotecache/v1/chains.go b/cache/remotecache/v1/chains.go index ee17a77f75b6..306e037f7f1c 100644 --- a/cache/remotecache/v1/chains.go +++ b/cache/remotecache/v1/chains.go @@ -1,6 +1,7 @@ package cacheimport import ( + "context" "strings" "sync" "time" @@ -75,7 +76,7 @@ func (c *CacheChains) normalize() error { return nil } -func (c *CacheChains) Marshal() (*CacheConfig, DescriptorProvider, error) { +func (c *CacheChains) Marshal(ctx context.Context) (*CacheConfig, DescriptorProvider, error) { if err := c.normalize(); err != nil { return nil, nil, err } @@ -87,7 +88,7 @@ func (c *CacheChains) Marshal() (*CacheConfig, DescriptorProvider, error) { } for _, it := range c.items { - if err := marshalItem(it, st); err != nil { + if err := marshalItem(ctx, it, st); err != nil { return nil, nil, err } } diff --git a/cache/remotecache/v1/chains_test.go b/cache/remotecache/v1/chains_test.go index 325465445dd0..7e2a2f525769 100644 --- a/cache/remotecache/v1/chains_test.go +++ b/cache/remotecache/v1/chains_test.go @@ -1,6 +1,7 @@ package cacheimport import ( + "context" "encoding/json" "testing" "time" @@ -33,7 +34,7 @@ func TestSimpleMarshal(t *testing.T) { addRecords() - cfg, _, err := cc.Marshal() + cfg, _, err := cc.Marshal(context.TODO()) require.NoError(t, err) require.Equal(t, len(cfg.Layers), 2) @@ -65,7 +66,7 @@ func TestSimpleMarshal(t *testing.T) { // adding same info again doesn't produce anything extra addRecords() - cfg2, descPairs, err := cc.Marshal() + cfg2, descPairs, err := cc.Marshal(context.TODO()) require.NoError(t, err) require.EqualValues(t, cfg, cfg2) @@ -78,13 +79,13 @@ func TestSimpleMarshal(t *testing.T) { err = Parse(dt, descPairs, newChains) require.NoError(t, err) - cfg3, _, err := cc.Marshal() + cfg3, _, err := cc.Marshal(context.TODO()) require.NoError(t, err) require.EqualValues(t, cfg, cfg3) // add extra item cc.Add(outputKey(dgst("bay"), 0)) - cfg, _, err = cc.Marshal() + cfg, _, err = cc.Marshal(context.TODO()) require.NoError(t, err) require.Equal(t, len(cfg.Layers), 2) diff --git a/cache/remotecache/v1/utils.go b/cache/remotecache/v1/utils.go index 889064a6c175..5ff0c4a796be 100644 --- a/cache/remotecache/v1/utils.go +++ b/cache/remotecache/v1/utils.go @@ -1,12 +1,14 @@ package cacheimport import ( + "context" "fmt" "sort" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/solver" digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -277,17 +279,27 @@ type marshalState struct { recordsByItem map[*item]int } -func marshalRemote(r *solver.Remote, state *marshalState) string { +func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) string { if len(r.Descriptors) == 0 { return "" } + + if cd, ok := r.Provider.(interface { + CheckDescriptor(context.Context, ocispecs.Descriptor) error + }); ok && len(r.Descriptors) > 0 { + for _, d := range r.Descriptors { + if cd.CheckDescriptor(ctx, d) != nil { + return "" + } + } + } var parentID string if len(r.Descriptors) > 1 { r2 := &solver.Remote{ Descriptors: r.Descriptors[:len(r.Descriptors)-1], Provider: r.Provider, } - parentID = marshalRemote(r2, state) + parentID = marshalRemote(ctx, r2, state) } desc := r.Descriptors[len(r.Descriptors)-1] @@ -318,7 +330,7 @@ func marshalRemote(r *solver.Remote, state *marshalState) string { return id } -func marshalItem(it *item, state *marshalState) error { +func marshalItem(ctx context.Context, it *item, state *marshalState) error { if _, ok := state.recordsByItem[it]; ok { return nil } @@ -330,7 +342,7 @@ func marshalItem(it *item, state *marshalState) error { for i, m := range it.links { for l := range m { - if err := marshalItem(l.src, state); err != nil { + if err := marshalItem(ctx, l.src, state); err != nil { return err } idx, ok := state.recordsByItem[l.src] @@ -345,7 +357,7 @@ func marshalItem(it *item, state *marshalState) error { } if it.result != nil { - id := marshalRemote(it.result, state) + id := marshalRemote(ctx, it.result, state) if id != "" { idx, ok := state.chainsByID[id] if !ok { diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index fc73723eba70..6a2a4e002b51 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -290,7 +290,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedResult, compressionopt solver.CompressionOpt, g session.Group) ([]byte, error) { if efl, ok := e.(interface { - ExportForLayers([]digest.Digest) ([]byte, error) + ExportForLayers(context.Context, []digest.Digest) ([]byte, error) }); ok { workerRef, ok := res.Sys().(*worker.WorkerRef) if !ok { @@ -317,7 +317,7 @@ func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedR return nil, err } - return efl.ExportForLayers(digests) + return efl.ExportForLayers(ctx, digests) } return nil, nil }