buildinfo: add build attributes and frontend

README.md

@@ -238,6 +238,7 @@ Keys supported by image output:
 * `compression-level=[value]`: compression level for gzip, estargz (0-9) and zstd (0-22)
 * `force-compression=true`: forcefully apply `compression` option to all layers (including already existing layers).
 * `buildinfo=[all,imageconfig,metadata,none]`: choose [build dependency](docs/build-repro.md#build-dependencies) version to export (default `all`).
+* `buildinfo-attrs=true`: inline build info attributes in [image config](docs/build-repro.md#image-config) (default `false`).
 
 If credentials are required, `buildctl` will attempt to read Docker configuration file `$DOCKER_CONFIG/config.json`.
 `$DOCKER_CONFIG` defaults to `~/.docker`.
@@ -434,8 +435,43 @@ The directory of the specified file must already exist and be writable.
 buildctl build ... --metadata-file metadata.json
 ```
 
+```shell
+jq '.' metadata.json
 ```
-{"containerimage.digest": "sha256:ea0cfb27fd41ea0405d3095880c1efa45710f5bcdddb7d7d5a7317ad4825ae14",...}
+```json
+{
+  "containerimage.buildinfo": {
+    "frontend": "dockerfile.v0",
+    "attrs": {
+      "context": "https://github.com/crazy-max/buildkit-buildsources-test.git#master",
+      "filename": "Dockerfile",
+      "source": "docker/dockerfile:master"
+    },
+    "sources": [
+      {
+        "type": "docker-image",
+        "ref": "docker.io/docker/buildx-bin:0.6.1@sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0",
+        "pin": "sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0"
+      },
+      {
+        "type": "docker-image",
+        "ref": "docker.io/library/alpine:3.13",
+        "pin": "sha256:026f721af4cf2843e07bba648e158fb35ecc876d822130633cc49f707f0fc88c"
+      }
+    ]
+  },
+  "containerimage.config.digest": "sha256:2937f66a9722f7f4a2df583de2f8cb97fc9196059a410e7f00072fc918930e66",
+  "containerimage.descriptor": {
+    "annotations": {
+      "config.digest": "sha256:2937f66a9722f7f4a2df583de2f8cb97fc9196059a410e7f00072fc918930e66",
+      "org.opencontainers.image.created": "2022-02-08T21:28:03Z"
+    },
+    "digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3",
+    "mediaType": "application/vnd.oci.image.manifest.v1+json",
+    "size": 506
+  },
+  "containerimage.digest": "sha256:19ffeab6f8bc9293ac2c3fdf94ebe28396254c993aea0b5a542cfb02e0883fa3"
+}
 ```
 
 ## Systemd socket activation

client/build_test.go

@@ -3,6 +3,8 @@ package client
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -15,6 +17,7 @@ import (
 	"time"
 
 	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/frontend/gateway/client"
 	gatewayapi "github.com/moby/buildkit/frontend/gateway/pb"
 	"github.com/moby/buildkit/identity"
@@ -23,6 +26,7 @@ import (
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
+	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/entitlements"
 	utilsystem "github.com/moby/buildkit/util/system"
 	"github.com/moby/buildkit/util/testutil/echoserver"
@@ -55,6 +59,7 @@ func TestClientGatewayIntegration(t *testing.T) {
 		testClientGatewayContainerExtraHosts,
 		testClientGatewayContainerSignal,
 		testWarnings,
+		testClientGatewayFrontendAttrs,
 	), integration.WithMirroredImages(integration.OfficialImages("busybox:latest")))
 
 	integration.Run(t, integration.TestFuncs(
@@ -1994,6 +1999,64 @@ func testClientGatewayContainerSignal(t *testing.T, sb integration.Sandbox) {
 	checkAllReleasable(t, c, sb, true)
 }
 
+// moby/buildkit#2476
+func testClientGatewayFrontendAttrs(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	fooattrval := "bar"
+	bazattrval := "fuu"
+
+	b := func(ctx context.Context, c client.Client) (*client.Result, error) {
+		st := llb.Image("busybox:latest").Run(
+			llb.ReadonlyRootFS(),
+			llb.Args([]string{"/bin/sh", "-c", `echo hello`}),
+		)
+		def, err := st.Marshal(sb.Context())
+		if err != nil {
+			return nil, err
+		}
+		res, err := c.Solve(ctx, client.SolveRequest{
+			Definition: def.ToPB(),
+			FrontendOpt: map[string]string{
+				"build-arg:foo": fooattrval,
+			},
+		})
+		require.NoError(t, err)
+		require.Contains(t, res.Metadata, exptypes.ExporterBuildInfo)
+
+		var bi binfotypes.BuildInfo
+		require.NoError(t, json.Unmarshal(res.Metadata[exptypes.ExporterBuildInfo], &bi))
+		require.Contains(t, bi.Attrs, "build-arg:foo")
+		bi.Attrs["build-arg:baz"] = &bazattrval
+
+		bmbi, err := json.Marshal(bi)
+		require.NoError(t, err)
+
+		res.AddMeta(exptypes.ExporterBuildInfo, bmbi)
+		return res, err
+	}
+
+	res, err := c.Build(sb.Context(), SolveOpt{}, "", b, nil)
+	require.NoError(t, err)
+
+	require.Contains(t, res.ExporterResponse, exptypes.ExporterBuildInfo)
+	decbi, err := base64.StdEncoding.DecodeString(res.ExporterResponse[exptypes.ExporterBuildInfo])
+	require.NoError(t, err)
+
+	var bi binfotypes.BuildInfo
+	require.NoError(t, json.Unmarshal(decbi, &bi))
+
+	require.Contains(t, bi.Attrs, "build-arg:foo")
+	require.Equal(t, &fooattrval, bi.Attrs["build-arg:foo"])
+	require.Contains(t, bi.Attrs, "build-arg:baz")
+	require.Equal(t, &bazattrval, bi.Attrs["build-arg:baz"])
+
+	checkAllReleasable(t, c, sb, true)
+}
+
 type nopCloser struct {
 	io.Writer
 }

client/client_test.go

@@ -7,6 +7,7 @@ import (
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
+	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
@@ -41,6 +42,7 @@ import (
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
+	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/moby/buildkit/util/testutil"
@@ -148,6 +150,8 @@ func TestIntegration(t *testing.T) {
 		testRmSymlink,
 		testMoveParentDir,
 		testBuildExportWithForeignLayer,
+		testBuildInfoExporter,
+		testBuildInfoInline,
 	)
 	tests = append(tests, diffOpTestCases()...)
 	integration.Run(t, tests, mirrors)
@@ -5038,6 +5042,152 @@ func testRelativeMountpoint(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, dt, []byte(id))
 }
 
+// moby/buildkit#2476
+func testBuildInfoExporter(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		st := llb.Image("busybox:latest").Run(
+			llb.Args([]string{"/bin/sh", "-c", `echo hello`}),
+		)
+		def, err := st.Marshal(sb.Context())
+		if err != nil {
+			return nil, err
+		}
+		return c.Solve(ctx, gateway.SolveRequest{
+			Definition:  def.ToPB(),
+			FrontendOpt: map[string]string{"build-arg:foo": "bar"},
+		})
+	}
+
+	registry, err := sb.NewRegistry()
+	if errors.Is(err, integration.ErrRequirements) {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+
+	res, err := c.Build(sb.Context(), SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type: ExporterImage,
+				Attrs: map[string]string{
+					"name": registry + "/buildkit/test-buildinfo:latest",
+					"push": "true",
+				},
+			},
+		},
+	}, "", frontend, nil)
+	require.NoError(t, err)
+
+	require.Contains(t, res.ExporterResponse, exptypes.ExporterBuildInfo)
+	decbi, err := base64.StdEncoding.DecodeString(res.ExporterResponse[exptypes.ExporterBuildInfo])
+	require.NoError(t, err)
+
+	var exbi binfotypes.BuildInfo
+	err = json.Unmarshal(decbi, &exbi)
+	require.NoError(t, err)
+
+	attrval := "bar"
+	require.Equal(t, exbi.Attrs, map[string]*string{"build-arg:foo": &attrval})
+	require.Equal(t, len(exbi.Sources), 1)
+	require.Equal(t, exbi.Sources[0].Type, binfotypes.SourceTypeDockerImage)
+	require.Equal(t, exbi.Sources[0].Ref, "docker.io/library/busybox:latest")
+}
+
+// moby/buildkit#2476
+func testBuildInfoInline(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	st := llb.Image("busybox:latest").Run(
+		llb.Args([]string{"/bin/sh", "-c", `echo hello`}),
+	)
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	registry, err := sb.NewRegistry()
+	if errors.Is(err, integration.ErrRequirements) {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+
+	cdAddress := sb.ContainerdAddress()
+	if cdAddress == "" {
+		t.Skip("rest of test requires containerd worker")
+	}
+
+	client, err := newContainerd(cdAddress)
+	require.NoError(t, err)
+	defer client.Close()
+
+	ctx := namespaces.WithNamespace(sb.Context(), "buildkit")
+
+	for _, tt := range []struct {
+		name       string
+		buildAttrs bool
+	}{{
+		"attrsEnabled",
+		true,
+	}, {
+		"attrsDisabled",
+		false,
+	}} {
+		t.Run(tt.name, func(t *testing.T) {
+			target := registry + "/buildkit/test-buildinfo:latest"
+
+			_, err = c.Solve(sb.Context(), def, SolveOpt{
+				Exports: []ExportEntry{
+					{
+						Type: ExporterImage,
+						Attrs: map[string]string{
+							"name":            target,
+							"push":            "true",
+							"buildinfo-attrs": strconv.FormatBool(tt.buildAttrs),
+						},
+					},
+				},
+				FrontendAttrs: map[string]string{
+					"build-arg:foo": "bar",
+				},
+			}, nil)
+			require.NoError(t, err)
+
+			img, err := client.GetImage(ctx, target)
+			require.NoError(t, err)
+
+			desc, err := img.Config(ctx)
+			require.NoError(t, err)
+
+			dt, err := content.ReadBlob(ctx, img.ContentStore(), desc)
+			require.NoError(t, err)
+
+			var config binfotypes.ImageConfig
+			err = json.Unmarshal(dt, &config)
+			require.NoError(t, err)
+
+			var bi binfotypes.BuildInfo
+			err = json.Unmarshal(config.BuildInfo, &bi)
+			require.NoError(t, err)
+
+			if tt.buildAttrs {
+				attrval := "bar"
+				require.Contains(t, bi.Attrs, "build-arg:foo")
+				require.Equal(t, bi.Attrs["build-arg:foo"], &attrval)
+			} else {
+				require.NotContains(t, bi.Attrs, "build-arg:foo")
+			}
+			require.Equal(t, len(bi.Sources), 1)
+			require.Equal(t, bi.Sources[0].Type, binfotypes.SourceTypeDockerImage)
+			require.Equal(t, bi.Sources[0].Ref, "docker.io/library/busybox:latest")
+		})
+	}
+}
+
 func tmpdir(appliers ...fstest.Applier) (string, error) {
 	tmpdir, err := ioutil.TempDir("", "buildkit-client")
 	if err != nil {