Merge pull request #4867 from AkihiroSuda/dev-0.13

[v0.13] cherry-picks

.github/workflows/.test.yml

@@ -194,6 +194,7 @@ jobs:
         with:
           name: test-reports-${{ env.TEST_REPORT_NAME }}
           path: ./bin/testreports
+          retention-days: 1
       -
         name: Dump context
         if: failure()

.github/workflows/buildkit.yml

@@ -113,6 +113,7 @@ jobs:
           name: buildkit-${{ env.PLATFORM_PAIR }}
           path: ${{ env.DESTDIR }}/*
           if-no-files-found: error
+          retention-days: 1
 
   test:
     uses: ./.github/workflows/.test.yml

.github/workflows/dockerd.yml

@@ -68,6 +68,7 @@ jobs:
           name: dockerd
           path: /tmp/moby/dockerd
           if-no-files-found: error
+          retention-days: 1
 
   test:
     runs-on: ubuntu-22.04

.github/workflows/test-os.yml

@@ -160,6 +160,7 @@ jobs:
         with:
           name: test-reports-${{ env.TEST_REPORT_NAME }}
           path: ./bin/testreports
+          retention-days: 1
       -
         name: Dump context
         if: failure()

exporter/containerimage/writer.go

@@ -419,11 +419,16 @@ func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCo
 	var divergedFromBase bool
 	for i, desc := range remoteDescriptors {
 		i, desc := i, desc
-		info, err := cs.Info(ctx, desc.Digest)
-		if err != nil {
-			return nil, err
+		// Usually we get non-empty diffID here, but if the content was ingested via a third-party containerd client,
+		// diffID here can be empty, and will be computed by the converter.
+		diffID := digest.Digest(desc.Annotations[labels.LabelUncompressed])
+		if diffID == "" {
+			info, err := cs.Info(ctx, desc.Digest)
+			if err != nil {
+				return nil, err
+			}
+			diffID = digest.Digest(info.Labels[labels.LabelUncompressed]) // can be still empty
 		}
-		diffID := digest.Digest(info.Labels[labels.LabelUncompressed]) // can be empty
 		var immDiffID digest.Digest
 		if !divergedFromBase && baseImg != nil && i < len(baseImg.RootFS.DiffIDs) {
 			immDiffID = baseImg.RootFS.DiffIDs[i]

frontend/dockerfile/dockerfile_test.go

@@ -6856,7 +6856,16 @@ func testReproSourceDateEpoch(t *testing.T, sb integration.Sandbox) {
 	tm := time.Date(2023, time.January, 10, 12, 34, 56, 0, time.UTC) // 1673354096
 	t.Logf("SOURCE_DATE_EPOCH=%d", tm.Unix())
 
-	dockerfile := []byte(`# The base image cannot be busybox, due to https://github.com/moby/buildkit/issues/3455
+	type testCase struct {
+		name           string
+		dockerfile     string
+		files          []fstest.Applier
+		expectedDigest string
+	}
+	testCases := []testCase{
+		{
+			name: "Basic",
+			dockerfile: `# The base image could not be busybox, due to https://github.com/moby/buildkit/issues/3455
 FROM amd64/debian:bullseye-20230109-slim
 RUN touch /foo
 RUN touch /foo.1
@@ -6867,7 +6876,20 @@ RUN touch -d '2030-01-01 12:34:56' /foo-2030.1
 RUN rm -f /foo.1
 RUN rm -f /foo-2010.1
 RUN rm -f /foo-2030.1
-`)
+`,
+			expectedDigest: "sha256:04e5d0cbee3317c79f50494cfeb4d8a728402a970ef32582ee47c62050037e3f",
+		},
+		{
+			// https://github.com/moby/buildkit/issues/4746
+			name: "CopyLink",
+			dockerfile: `FROM amd64/debian:bullseye-20230109-slim
+COPY --link foo foo
+`,
+			files:          []fstest.Applier{fstest.CreateFile("foo", []byte("foo"), 0600)},
+			expectedDigest: "sha256:9f75e4bdbf3d825acb36bb603ddef4a25742afb8ccb674763ffc611ae047d8a6",
+		},
+	}
+
 	// https://explore.ggcr.dev/?image=amd64%2Fdebian%3Abullseye-20230109-slim
 	baseImageLayers := []digest.Digest{
 		"sha256:8740c948ffd4c816ea7ca963f99ca52f4788baa23f228da9581a9ea2edd3fcd7",
@@ -6877,88 +6899,98 @@ RUN rm -f /foo-2030.1
 		timeMustParse(t, time.RFC3339Nano, "2023-01-11T02:34:44.829692296Z"),
 	}
 
-	const expectedDigest = "sha256:04e5d0cbee3317c79f50494cfeb4d8a728402a970ef32582ee47c62050037e3f"
-
-	dir := integration.Tmpdir(
-		t,
-		fstest.CreateFile("Dockerfile", dockerfile, 0600),
-	)
-
 	ctx := sb.Context()
 	c, err := client.New(ctx, sb.Address())
 	require.NoError(t, err)
 	defer c.Close()
 
-	target := registry + "/buildkit/testreprosourcedateepoch:" + fmt.Sprintf("%d", tm.Unix())
-	solveOpt := client.SolveOpt{
-		FrontendAttrs: map[string]string{
-			"build-arg:SOURCE_DATE_EPOCH": fmt.Sprintf("%d", tm.Unix()),
-			"platform":                    "linux/amd64",
-		},
-		LocalMounts: map[string]fsutil.FS{
-			dockerui.DefaultLocalNameDockerfile: dir,
-			dockerui.DefaultLocalNameContext:    dir,
-		},
-		Exports: []client.ExportEntry{
-			{
-				Type: client.ExporterImage,
-				Attrs: map[string]string{
-					"name":              target,
-					"push":              "true",
-					"oci-mediatypes":    "true",
-					"rewrite-timestamp": "true",
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := integration.Tmpdir(
+				t,
+				append([]fstest.Applier{fstest.CreateFile("Dockerfile", []byte(tc.dockerfile), 0600)}, tc.files...)...,
+			)
+
+			target := registry + "/buildkit/testreprosourcedateepoch-" + strings.ToLower(tc.name) + ":" + fmt.Sprintf("%d", tm.Unix())
+			solveOpt := client.SolveOpt{
+				FrontendAttrs: map[string]string{
+					"build-arg:SOURCE_DATE_EPOCH": fmt.Sprintf("%d", tm.Unix()),
+					"platform":                    "linux/amd64",
 				},
-			},
-		},
-		CacheExports: []client.CacheOptionsEntry{
-			{
-				Type: "registry",
-				Attrs: map[string]string{
-					"ref":            target + "-cache",
-					"oci-mediatypes": "true",
-					"image-manifest": "true",
+				LocalMounts: map[string]fsutil.FS{
+					dockerui.DefaultLocalNameDockerfile: dir,
+					dockerui.DefaultLocalNameContext:    dir,
 				},
-			},
-		},
-	}
-	_, err = f.Solve(ctx, c, solveOpt, nil)
-	require.NoError(t, err)
+				Exports: []client.ExportEntry{
+					{
+						Type: client.ExporterImage,
+						Attrs: map[string]string{
+							"name":              target,
+							"push":              "true",
+							"oci-mediatypes":    "true",
+							"rewrite-timestamp": "true",
+						},
+					},
+				},
+				CacheExports: []client.CacheOptionsEntry{
+					{
+						Type: "registry",
+						Attrs: map[string]string{
+							"ref":            target + "-cache",
+							"oci-mediatypes": "true",
+							"image-manifest": "true",
+						},
+					},
+				},
+			}
+			_, err = f.Solve(ctx, c, solveOpt, nil)
+			require.NoError(t, err)
 
-	desc, manifest, img := readImage(t, ctx, target)
-	_, cacheManifest, _ := readImage(t, ctx, target+"-cache")
-	t.Log("The digest may change depending on the BuildKit version, the snapshotter configuration, etc.")
-	require.Equal(t, expectedDigest, desc.Digest.String())
+			desc, manifest, img := readImage(t, ctx, target)
+			_, cacheManifest, _ := readImage(t, ctx, target+"-cache")
+			t.Log("The digest may change depending on the BuildKit version, the snapshotter configuration, etc.")
+			require.Equal(t, tc.expectedDigest, desc.Digest.String())
 
-	// Image history from the base config must remain immutable
-	for i, tm := range baseImageHistoryTimestamps {
-		require.True(t, img.History[i].Created.Equal(tm))
-	}
+			// Image history from the base config must remain immutable
+			for i, tm := range baseImageHistoryTimestamps {
+				require.True(t, img.History[i].Created.Equal(tm))
+			}
 
-	// Image layers, *except the base layers*, must have rewritten-timestamp
-	for i, l := range manifest.Layers {
-		if i < len(baseImageLayers) {
-			require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
-			require.Equal(t, baseImageLayers[i], l.Digest)
-		} else {
-			require.Equal(t, fmt.Sprintf("%d", tm.Unix()), l.Annotations["buildkit/rewritten-timestamp"])
-		}
-	}
-	// Cache layers must *not* have rewritten-timestamp
-	for _, l := range cacheManifest.Layers {
-		require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
-	}
+			// Image layers, *except the base layers*, must have rewritten-timestamp
+			for i, l := range manifest.Layers {
+				if i < len(baseImageLayers) {
+					require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
+					require.Equal(t, baseImageLayers[i], l.Digest)
+				} else {
+					require.Equal(t, fmt.Sprintf("%d", tm.Unix()), l.Annotations["buildkit/rewritten-timestamp"])
+				}
+			}
+			// Cache layers must *not* have rewritten-timestamp
+			for _, l := range cacheManifest.Layers {
+				require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
+			}
 
-	// Build again, but without rewrite-timestamp
-	solveOpt2 := solveOpt
-	delete(solveOpt2.Exports[0].Attrs, "rewrite-timestamp")
-	_, err = f.Solve(ctx, c, solveOpt2, nil)
-	require.NoError(t, err)
-	_, manifest2, img2 := readImage(t, ctx, target)
-	for i, tm := range baseImageHistoryTimestamps {
-		require.True(t, img2.History[i].Created.Equal(tm))
-	}
-	for _, l := range manifest2.Layers {
-		require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
+			// Build again, after pruning the base image layer cache.
+			// For testing https://github.com/moby/buildkit/issues/4746
+			ensurePruneAll(t, c, sb)
+			_, err = f.Solve(ctx, c, solveOpt, nil)
+			require.NoError(t, err)
+			descAfterPrune, _, _ := readImage(t, ctx, target)
+			require.Equal(t, desc.Digest.String(), descAfterPrune.Digest.String())
+
+			// Build again, but without rewrite-timestamp
+			solveOpt2 := solveOpt
+			delete(solveOpt2.Exports[0].Attrs, "rewrite-timestamp")
+			_, err = f.Solve(ctx, c, solveOpt2, nil)
+			require.NoError(t, err)
+			_, manifest2, img2 := readImage(t, ctx, target)
+			for i, tm := range baseImageHistoryTimestamps {
+				require.True(t, img2.History[i].Created.Equal(tm))
+			}
+			for _, l := range manifest2.Layers {
+				require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
+			}
+		})
 	}
 }
 
@@ -6978,6 +7010,14 @@ func readImage(t *testing.T, ctx context.Context, ref string) (ocispecs.Descript
 	require.NoError(t, json.Unmarshal(dt, &manifest))
 	imgDt, err := content.ReadBlob(ctx, provider, manifest.Config)
 	require.NoError(t, err)
+	// Verify that all the layer blobs are present
+	for _, layer := range manifest.Layers {
+		layerRA, err := provider.ReaderAt(ctx, layer)
+		require.NoError(t, err)
+		layerDigest, err := layer.Digest.Algorithm().FromReader(content.NewReader(layerRA))
+		require.NoError(t, err)
+		require.Equal(t, layer.Digest, layerDigest)
+	}
 	var img ocispecs.Image
 	require.NoError(t, json.Unmarshal(imgDt, &img))
 	return desc, manifest, img

go.mod

@@ -91,11 +91,11 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v1.21.0
 	go.opentelemetry.io/otel/trace v1.21.0
 	go.opentelemetry.io/proto/otlp v1.0.0
-	golang.org/x/crypto v0.17.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/mod v0.13.0
-	golang.org/x/net v0.18.0
+	golang.org/x/net v0.23.0
 	golang.org/x/sync v0.5.0
-	golang.org/x/sys v0.16.0
+	golang.org/x/sys v0.18.0
 	golang.org/x/time v0.3.0
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b
 	google.golang.org/grpc v1.59.0

go.sum

@@ -473,8 +473,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -497,8 +497,8 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
 golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
@@ -537,11 +537,11 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

util/converter/tarconverter/tarconverter.go

@@ -8,6 +8,7 @@ import (
 type HeaderConverter func(*tar.Header)
 
 // NewReader returns a reader that applies headerConverter.
+// srcContent is drained until hitting EOF.
 // Forked from https://github.com/moby/moby/blob/v24.0.6/pkg/archive/copy.go#L308-L373 .
 func NewReader(srcContent io.Reader, headerConverter HeaderConverter) io.ReadCloser {
 	rebased, w := io.Pipe()
@@ -21,7 +22,14 @@ func NewReader(srcContent io.Reader, headerConverter HeaderConverter) io.ReadClo
 			if err == io.EOF {
 				// Signals end of archive.
 				rebasedTar.Close()
-				w.Close()
+				// drain the reader into io.Discard, until hitting EOF
+				// https://github.com/moby/buildkit/pull/4807#discussion_r1544621787
+				_, err = io.Copy(io.Discard, srcContent)
+				if err != nil {
+					w.CloseWithError(err)
+				} else {
+					w.Close()
+				}
 				return
 			}
 			if err != nil {