[0.25] cherry-picks for v0.25.1

cache/remotecache/azblob/importer.go

@@ -11,6 +11,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/pkg/labels"
+	cerrdefs "github.com/containerd/errdefs"
 	"github.com/moby/buildkit/cache/remotecache"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
 	"github.com/moby/buildkit/session"
@@ -214,7 +215,7 @@ type ciProvider struct {
 
 func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
 	if dgst != p.desc.Digest {
-		return content.Info{}, errors.Errorf("content not found %s", dgst)
+		return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
 	}
 
 	if p.checked {
@@ -234,7 +235,7 @@ func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info
 	}
 
 	if !exists {
-		return content.Info{}, errors.Errorf("blob %s not found", dgst)
+		return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
 	}
 
 	p.checked = true

cache/remotecache/gha/gha.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/pkg/labels"
+	cerrdefs "github.com/containerd/errdefs"
 	"github.com/moby/buildkit/cache/remotecache"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
 	"github.com/moby/buildkit/session"
@@ -440,7 +441,7 @@ type ciProvider struct {
 
 func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
 	if dgst != p.desc.Digest {
-		return content.Info{}, errors.Errorf("content not found %s", dgst)
+		return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
 	}
 
 	if _, err := p.loadEntry(ctx, p.desc); err != nil {
@@ -465,7 +466,7 @@ func (p *ciProvider) loadEntry(ctx context.Context, desc ocispecs.Descriptor) (*
 		return nil, err
 	}
 	if ce == nil {
-		return nil, errors.Errorf("blob %s not found", desc.Digest)
+		return nil, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", desc.Digest)
 	}
 	if p.entries == nil {
 		p.entries = make(map[digest.Digest]*actionscache.Entry)

cache/remotecache/v1/chains.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/containerd/containerd/v2/core/content"
+	cerrdefs "github.com/containerd/errdefs"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	digest "github.com/opencontainers/go-digest"
@@ -246,7 +247,7 @@ func (p DescriptorProviderPair) Info(ctx context.Context, dgst digest.Digest) (c
 		return p.InfoProvider.Info(ctx, dgst)
 	}
 	if dgst != p.Descriptor.Digest {
-		return content.Info{}, errors.Errorf("content not found %s", dgst)
+		return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst)
 	}
 	return content.Info{
 		Digest: p.Descriptor.Digest,

docs/buildkitd.toml.md

@@ -21,6 +21,11 @@ trace = true
 root = "/var/lib/buildkit"
 # insecure-entitlements allows insecure entitlements, disabled by default.
 insecure-entitlements = [ "network.host", "security.insecure", "device" ]
+# provenanceEnvDir is the directory where extra config is loaded that is added
+# to the provenance of builds:
+# slsa v0.2: invocation.environment.*
+# slsa v1: buildDefinition.internalParameters.*
+provenanceEnvDir = "/etc/buildkit/provenance.d"
 
 [log]
   # log formatter: json or text

hack/images

@@ -108,7 +108,7 @@ done
 
 nocacheFilterFlag=""
 if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then
-  nocacheFilterFlag="--no-cache-filter=buildkit-export,gobuild-base,rootless"
+  nocacheFilterFlag="--no-cache-filter=buildkit-export-alpine,buildkit-export-ubuntu,gobuild-base,rootless"
 fi
 
 buildxCmd build --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" --build-arg BUILDKITD_TAGS --build-arg BUILDKIT_DEBUG --build-arg EXPORT_BASE $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \

solver/exporter.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"slices"
 
+	cerrdefs "github.com/containerd/errdefs"
 	digest "github.com/opencontainers/go-digest"
 )
 
@@ -189,24 +190,28 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		if (remote == nil || opt.CompressionOpt != nil) && opt.Mode != CacheExportModeRemoteOnly {
 			res, err := cm.results.Load(ctx, res)
 			if err != nil {
-				return nil, err
-			}
-			remotes, err := opt.ResolveRemotes(ctx, res)
-			if err != nil {
-				return nil, err
-			}
-			res.Release(context.TODO())
-			if remote == nil && len(remotes) > 0 {
-				remote, remotes = remotes[0], remotes[1:] // pop the first element
-			}
-			if opt.CompressionOpt != nil {
-				for _, r := range remotes { // record all remaining remotes as well
-					results = append(results, CacheExportResult{
-						CreatedAt:  v.CreatedAt,
-						Result:     r,
-						EdgeVertex: k.vtx,
-						EdgeIndex:  k.output,
-					})
+				if !errors.Is(err, cerrdefs.ErrNotFound) {
+					return nil, err
+				}
+				remote = nil
+			} else {
+				remotes, err := opt.ResolveRemotes(ctx, res)
+				if err != nil {
+					return nil, err
+				}
+				res.Release(context.TODO())
+				if remote == nil && len(remotes) > 0 {
+					remote, remotes = remotes[0], remotes[1:] // pop the first element
+				}
+				if opt.CompressionOpt != nil {
+					for _, r := range remotes { // record all remaining remotes as well
+						results = append(results, CacheExportResult{
+							CreatedAt:  v.CreatedAt,
+							Result:     r,
+							EdgeVertex: k.vtx,
+							EdgeIndex:  k.output,
+						})
+					}
 				}
 			}
 		}
@@ -232,7 +237,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		for _, dep := range deps {
 			rec, err := dep.CacheKey.Exporter.ExportTo(ctx, t, opt)
 			if err != nil {
-				return nil, err
+				continue
 			}
 			for _, r := range rec {
 				srcs[i] = append(srcs[i], CacheLink{Src: r, Selector: string(dep.Selector)})
@@ -244,7 +249,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		for _, de := range e.edge.secondaryExporters {
 			recs, err := de.cacheKey.CacheKey.Exporter.ExportTo(mainCtx, t, opt)
 			if err != nil {
-				return nil, nil
+				continue
 			}
 			for _, r := range recs {
 				srcs[de.index] = append(srcs[de.index], CacheLink{Src: r, Selector: de.cacheKey.Selector.String()})
@@ -261,6 +266,14 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		}
 	}
 
+	// validate deps are present
+	for _, deps := range srcs {
+		if len(deps) == 0 {
+			res[e] = nil
+			return res[e], nil
+		}
+	}
+
 	if v != nil && len(deps) == 0 {
 		cm := v.cacheManager
 		key := cm.getID(v.key)

solver/llbsolver/provenance/types/types.go

@@ -1,6 +1,8 @@
 package types
 
 import (
+	"encoding/json"
+	"maps"
 	"slices"
 
 	slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/common"
@@ -311,3 +313,95 @@ func (p *ProvenancePredicateSLSA02) ConvertToSLSA1() *ProvenancePredicateSLSA1 {
 		RunDetails:      runDetails,
 	}
 }
+
+// MarshalJSON flattens ProvenanceCustomEnv into top level.
+func (p ProvenanceInternalParametersSLSA1) MarshalJSON() ([]byte, error) {
+	type Alias ProvenanceInternalParametersSLSA1
+	base, err := json.Marshal(Alias(p))
+	if err != nil {
+		return nil, err
+	}
+	var m map[string]any
+	if err := json.Unmarshal(base, &m); err != nil {
+		return nil, err
+	}
+	maps.Copy(m, p.ProvenanceCustomEnv)
+	delete(m, "ProvenanceCustomEnv")
+	return json.Marshal(m)
+}
+
+// UnmarshalJSON fills both struct fields and flattened custom env.
+func (p *ProvenanceInternalParametersSLSA1) UnmarshalJSON(data []byte) error {
+	var m map[string]any
+	if err := json.Unmarshal(data, &m); err != nil {
+		return err
+	}
+
+	type Alias ProvenanceInternalParametersSLSA1
+	var a Alias
+	if err := json.Unmarshal(data, &a); err != nil {
+		return err
+	}
+
+	// Unmarshal known struct again to identify its keys
+	structBytes, err := json.Marshal(a)
+	if err != nil {
+		return err
+	}
+	var known map[string]any
+	if err := json.Unmarshal(structBytes, &known); err != nil {
+		return err
+	}
+
+	for k := range known {
+		delete(m, k)
+	}
+
+	*p = ProvenanceInternalParametersSLSA1(a)
+	p.ProvenanceCustomEnv = m
+	return nil
+}
+
+func (p Environment) MarshalJSON() ([]byte, error) {
+	type Alias Environment
+	base, err := json.Marshal(Alias(p))
+	if err != nil {
+		return nil, err
+	}
+	var m map[string]any
+	if err := json.Unmarshal(base, &m); err != nil {
+		return nil, err
+	}
+	maps.Copy(m, p.ProvenanceCustomEnv)
+	delete(m, "ProvenanceCustomEnv")
+	return json.Marshal(m)
+}
+
+func (p *Environment) UnmarshalJSON(data []byte) error {
+	var m map[string]any
+	if err := json.Unmarshal(data, &m); err != nil {
+		return err
+	}
+
+	type Alias Environment
+	var a Alias
+	if err := json.Unmarshal(data, &a); err != nil {
+		return err
+	}
+	// Unmarshal known struct again to identify its keys
+	structBytes, err := json.Marshal(a)
+	if err != nil {
+		return err
+	}
+	var known map[string]any
+	if err := json.Unmarshal(structBytes, &known); err != nil {
+		return err
+	}
+
+	for k := range known {
+		delete(m, k)
+	}
+	*p = Environment(a)
+	p.ProvenanceCustomEnv = m
+	return nil
+}

solver/llbsolver/provenance/types/types_test.go

@@ -0,0 +1,73 @@
+package types
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMarsalBuildDefinitionSLSA1(t *testing.T) {
+	inp := `{
+"buildType": "btype1",
+"externalParameters": {
+	"configSource": {},
+	"request": {}
+},
+"internalParameters": {
+	"builderPlatform": "linux/amd64",
+		"foo": "bar",
+		"abc": 123,
+		"def": {"one": 1}
+	}
+}`
+
+	var def ProvenanceBuildDefinitionSLSA1
+	err := json.Unmarshal([]byte(inp), &def)
+	require.NoError(t, err)
+
+	require.Equal(t, "btype1", def.BuildType)
+	require.Equal(t, "linux/amd64", def.InternalParameters.BuilderPlatform)
+	require.Equal(t, "bar", def.InternalParameters.ProvenanceCustomEnv["foo"])
+	require.InEpsilon(t, float64(123), def.InternalParameters.ProvenanceCustomEnv["abc"], 0.001)
+	require.Equal(t, map[string]any{"one": float64(1)}, def.InternalParameters.ProvenanceCustomEnv["def"])
+
+	out, err := json.Marshal(def)
+	require.NoError(t, err)
+
+	require.JSONEq(t, inp, string(out))
+}
+
+func TestMarshalInvocation(t *testing.T) {
+	inp := `{
+	"configSource": {
+		"uri": "git+https://github.com/example/repo.git"
+	},
+	"parameters": {
+		"frontend": "dockerfile.v0"
+	},
+	"environment": {
+		"platform": "linux/amd64",
+		"buildkit": "v0.10.3",
+		"custom": {
+			"foo": "bar"
+		},
+		"bar": [1,2,3]
+	}
+}`
+
+	var inv ProvenanceInvocationSLSA02
+	err := json.Unmarshal([]byte(inp), &inv)
+	require.NoError(t, err)
+
+	require.Equal(t, "git+https://github.com/example/repo.git", inv.ConfigSource.URI)
+	require.Equal(t, "dockerfile.v0", inv.Parameters.Frontend)
+	require.Equal(t, "linux/amd64", inv.Environment.Platform)
+	require.Equal(t, "v0.10.3", inv.Environment.ProvenanceCustomEnv["buildkit"])
+	require.Equal(t, "bar", inv.Environment.ProvenanceCustomEnv["custom"].(map[string]any)["foo"])
+	require.Equal(t, []any{float64(1), float64(2), float64(3)}, inv.Environment.ProvenanceCustomEnv["bar"])
+	out, err := json.Marshal(inv)
+	require.NoError(t, err)
+
+	require.JSONEq(t, inp, string(out))
+}

solver/llbsolver/solver.go

@@ -847,6 +847,7 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter
 	eg, ctx := errgroup.WithContext(ctx)
 	resps := make([]map[string]string, len(exporters))
 	descs := make([]exporter.DescriptorReference, len(exporters))
+	var inlineCacheMu sync.Mutex
 	for i, exp := range exporters {
 		i, exp := i, exp
 		eg.Go(func() error {
@@ -865,6 +866,8 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter
 					}
 				}
 				inlineCache := exptypes.InlineCache(func(ctx context.Context) (*result.Result[*exptypes.InlineCacheEntry], error) {
+					inlineCacheMu.Lock() // ensure only one inline cache exporter runs at a time
+					defer inlineCacheMu.Unlock()
 					return runInlineCacheExporter(ctx, exp, inlineCacheExporter, job, cached)
 				})