Merge pull request #2311 from crazy-max/buildsources

Generate and embed build sources

README.md

@@ -59,6 +59,7 @@ You don't need to read this document unless you want to use the full-featured st
     - [Local directory](#local-directory-1)
     - [GitHub Actions cache (experimental)](#github-actions-cache-experimental)
   - [Consistent hashing](#consistent-hashing)
+- [Metadata](#metadata)
 - [Systemd socket activation](#systemd-socket-activation)
 - [Expose BuildKit as a TCP service](#expose-buildkit-as-a-tcp-service)
   - [Load balancing](#load-balancing)
@@ -232,6 +233,7 @@ Keys supported by image output:
 * `name-canonical=true`: add additional canonical name `name@<digest>`
 * `compression=[uncompressed,gzip,estargz,zstd]`: choose compression type for layers newly created and cached, gzip is default value. estargz should be used with `oci-mediatypes=true`.
 * `force-compression=true`: forcefully apply `compression` option to all layers (including already existing layers).
+* `buildinfo=[all,imageconfig,metadata,none]`: choose [build dependency](docs/build-repro.md#build-dependencies) version to export (default `all`).
 
 If credentials are required, `buildctl` will attempt to read Docker configuration file `$DOCKER_CONFIG/config.json`.
 `$DOCKER_CONFIG` defaults to `~/.docker`.
@@ -416,7 +418,7 @@ To output build metadata such as the image digest, pass the `--metadata-file` fl
 The metadata will be written as a JSON object to the specified file.
 The directory of the specified file must already exist and be writable.
 
-```
+```bash
 buildctl build ... --metadata-file metadata.json
 ```
 

docs/build-repro.md

@@ -0,0 +1,83 @@
+# Build reproducibility
+
+## Build dependencies
+
+Build dependencies are generated when your image has been built. These
+dependencies include versions of used images, git repositories and HTTP URLs
+used by LLB `Source` operation.
+
+By default, the build dependencies are embedded in the image configuration and
+also available in the solver response. The export mode can be refined with
+the [`buildinfo` attribute](../README.md#imageregistry).
+
+### Image config
+
+A new field similar to the one for inline cache has been added to the image
+configuration to embed build dependencies:
+
+```text
+"moby.buildkit.buildinfo.v1": <base64>
+```
+
+The structure is base64 encoded and has the following format when decoded:
+
+```json
+{
+  "sources": [
+    {
+      "type": "docker-image",
+      "ref": "docker.io/docker/buildx-bin:0.6.1@sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0",
+      "pin": "sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0"
+    },
+    {
+      "type": "docker-image",
+      "ref": "docker.io/library/alpine:3.13",
+      "pin": "sha256:1d30d1ba3cb90962067e9b29491fbd56997979d54376f23f01448b5c5cd8b462"
+    },
+    {
+      "type": "git",
+      "ref": "https://github.com/crazy-max/buildkit-buildsources-test.git#master",
+      "pin": "259a5aa5aa5bb3562d12cc631fe399f4788642c1"
+    },
+    {
+      "type": "http",
+      "ref": "https://raw.githubusercontent.com/moby/moby/master/README.md",
+      "pin": "sha256:419455202b0ef97e480d7f8199b26a721a417818bc0e2d106975f74323f25e6c"
+    }
+  ]
+}
+```
+
+* `type` defines the source type (`docker-image`, `git` or `http`).
+* `ref` is the reference of the source.
+* `pin` is the source digest.
+
+### Exporter response (metadata)
+
+The solver response (`ExporterResponse`) also contains a new key
+`containerimage.buildinfo` with the same structure as image config encoded in
+base64:
+
+```json
+{
+  "ExporterResponse": {
+    "containerimage.buildinfo": "<base64>",
+    "containerimage.digest": "sha256:...",
+    "image.name": "..."
+  }
+}
+```
+
+If multi-platforms are specified, they will be suffixed with the corresponding
+platform:
+
+```json
+{
+  "ExporterResponse": {
+    "containerimage.buildinfo/linux/amd64": "<base64>",
+    "containerimage.buildinfo/linux/arm64": "<base64>",
+    "containerimage.digest": "sha256:...",
+    "image.name": "..."
+  }
+}
+```

exporter/containerimage/export.go

@@ -19,6 +19,7 @@ import (
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
+	"github.com/moby/buildkit/util/buildinfo"
 	"github.com/moby/buildkit/util/compression"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/leaseutil"
@@ -40,6 +41,7 @@ const (
 	keyNameCanonical    = "name-canonical"
 	keyLayerCompression = "compression"
 	keyForceCompression = "force-compression"
+	keyBuildInfo        = "buildinfo"
 	ociTypes            = "oci-mediatypes"
 )
 
@@ -68,6 +70,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 	i := &imageExporterInstance{
 		imageExporter:    e,
 		layerCompression: compression.Default,
+		buildInfoMode:    buildinfo.ExportDefault,
 	}
 
 	var esgz bool
@@ -161,6 +164,15 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
 			}
 			i.forceCompression = b
+		case keyBuildInfo:
+			if v == "" {
+				continue
+			}
+			bimode, err := buildinfo.ParseExportMode(v)
+			if err != nil {
+				return nil, err
+			}
+			i.buildInfoMode = bimode
 		default:
 			if i.meta == nil {
 				i.meta = make(map[string][]byte)
@@ -187,6 +199,7 @@ type imageExporterInstance struct {
 	danglingPrefix   string
 	layerCompression compression.Type
 	forceCompression bool
+	buildInfoMode    buildinfo.ExportMode
 	meta             map[string][]byte
 }
 
@@ -208,7 +221,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source,
 	}
 	defer done(context.TODO())
 
-	desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.forceCompression, sessionID)
+	desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.buildInfoMode, e.forceCompression, sessionID)
 	if err != nil {
 		return nil, err
 	}
@@ -217,6 +230,15 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source,
 		e.opt.ImageWriter.ContentStore().Delete(context.TODO(), desc.Digest)
 	}()
 
+	if e.buildInfoMode&buildinfo.ExportMetadata == 0 {
+		for k := range src.Metadata {
+			if !strings.HasPrefix(k, exptypes.ExporterBuildInfo) {
+				continue
+			}
+			delete(src.Metadata, k)
+		}
+	}
+
 	resp := make(map[string]string)
 
 	if n, ok := src.Metadata["image.name"]; e.targetName == "*" && ok {

exporter/containerimage/exptypes/types.go

@@ -1,6 +1,7 @@
 package exptypes
 
 import (
+	srctypes "github.com/moby/buildkit/source/types"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 )
@@ -11,6 +12,7 @@ const (
 	ExporterImageConfigKey       = "containerimage.config"
 	ExporterImageConfigDigestKey = "containerimage.config.digest"
 	ExporterInlineCache          = "containerimage.inlinecache"
+	ExporterBuildInfo            = "containerimage.buildinfo"
 	ExporterPlatformsKey         = "refs.platforms"
 )
 
@@ -24,3 +26,29 @@ type Platform struct {
 	ID       string
 	Platform ocispecs.Platform
 }
+
+// BuildInfo defines build dependencies that will be added to image config as
+// moby.buildkit.buildinfo.v1 key and returned in solver ExporterResponse as
+// ExporterBuildInfo key.
+type BuildInfo struct {
+	// Type defines the BuildInfoType source type (docker-image, git, http).
+	Type BuildInfoType `json:"type,omitempty"`
+	// Ref is the reference of the source.
+	Ref string `json:"ref,omitempty"`
+	// Alias is a special field used to match with the actual source ref
+	// because frontend might have already transformed a string user typed
+	// before generating LLB.
+	Alias string `json:"alias,omitempty"`
+	// Pin is the source digest.
+	Pin string `json:"pin,omitempty"`
+}
+
+// BuildInfoType contains source type.
+type BuildInfoType string
+
+// List of source types.
+const (
+	BuildInfoTypeDockerImage BuildInfoType = srctypes.DockerImageScheme
+	BuildInfoTypeGit         BuildInfoType = srctypes.GitScheme
+	BuildInfoTypeHTTP        BuildInfoType = srctypes.HTTPScheme
+)

exporter/containerimage/writer.go

@@ -8,11 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/moby/buildkit/util/bklog"
-	"github.com/moby/buildkit/util/tracing"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/trace"
-
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/images"
@@ -23,13 +18,18 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/util/bklog"
+	"github.com/moby/buildkit/util/buildinfo"
 	"github.com/moby/buildkit/util/compression"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/system"
+	"github.com/moby/buildkit/util/tracing"
 	digest "github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -48,7 +48,7 @@ type ImageWriter struct {
 	opt WriterOpt
 }
 
-func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool, compressionType compression.Type, forceCompression bool, sessionID string) (*ocispecs.Descriptor, error) {
+func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool, compressionType compression.Type, buildInfoMode buildinfo.ExportMode, forceCompression bool, sessionID string) (*ocispecs.Descriptor, error) {
 	platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey]
 
 	if len(inp.Refs) > 0 && !ok {
@@ -60,14 +60,21 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool
 		if err != nil {
 			return nil, err
 		}
-		mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], &remotes[0], oci, inp.Metadata[exptypes.ExporterInlineCache])
+
+		var buildInfo []byte
+		if buildInfoMode&buildinfo.ExportImageConfig > 0 {
+			buildInfo = inp.Metadata[exptypes.ExporterBuildInfo]
+		}
+
+		mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], &remotes[0], oci, inp.Metadata[exptypes.ExporterInlineCache], buildInfo)
 		if err != nil {
 			return nil, err
 		}
 		if mfstDesc.Annotations == nil {
 			mfstDesc.Annotations = make(map[string]string)
 		}
 		mfstDesc.Annotations[exptypes.ExporterConfigDigestKey] = configDesc.Digest.String()
+
 		return mfstDesc, nil
 	}
 
@@ -119,8 +126,14 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool
 			return nil, errors.Errorf("failed to find ref for ID %s", p.ID)
 		}
 		config := inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, p.ID)]
+		inlineCache := inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterInlineCache, p.ID)]
+
+		var buildInfo []byte
+		if buildInfoMode&buildinfo.ExportImageConfig > 0 {
+			buildInfo = inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, p.ID)]
+		}
 
-		desc, _, err := ic.commitDistributionManifest(ctx, r, config, &remotes[remotesMap[p.ID]], oci, inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterInlineCache, p.ID)])
+		desc, _, err := ic.commitDistributionManifest(ctx, r, config, &remotes[remotesMap[p.ID]], oci, inlineCache, buildInfo)
 		if err != nil {
 			return nil, err
 		}
@@ -184,7 +197,7 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, compressionType compres
 	return out, err
 }
 
-func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache.ImmutableRef, config []byte, remote *solver.Remote, oci bool, inlineCache []byte) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) {
+func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache.ImmutableRef, config []byte, remote *solver.Remote, oci bool, inlineCache []byte, buildInfo []byte) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) {
 	if len(config) == 0 {
 		var err error
 		config, err = emptyImageConfig()
@@ -206,7 +219,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache
 
 	remote, history = normalizeLayersAndHistory(ctx, remote, history, ref, oci)
 
-	config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache)
+	config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache, buildInfo)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -346,7 +359,7 @@ func parseHistoryFromConfig(dt []byte) ([]ocispecs.History, error) {
 	return config.History, nil
 }
 
-func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte) ([]byte, error) {
+func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte, buildInfo []byte) ([]byte, error) {
 	m := map[string]json.RawMessage{}
 	if err := json.Unmarshal(dt, &m); err != nil {
 		return nil, errors.Wrap(err, "failed to parse image config for patch")
@@ -391,6 +404,16 @@ func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs
 		m["moby.buildkit.cache.v0"] = dt
 	}
 
+	if buildInfo != nil {
+		dt, err := json.Marshal(buildInfo)
+		if err != nil {
+			return nil, err
+		}
+		m[buildinfo.ImageConfigField] = dt
+	} else if _, ok := m[buildinfo.ImageConfigField]; ok {
+		delete(m, buildinfo.ImageConfigField)
+	}
+
 	dt, err = json.Marshal(m)
 	return dt, errors.Wrap(err, "failed to marshal config after patch")
 }