Skip to content

Commit

Permalink
Merge pull request #4867 from AkihiroSuda/dev-0.13
Browse files Browse the repository at this point in the history
[v0.13] cherry-picks
  • Loading branch information
tonistiigi authored Apr 23, 2024
2 parents 2ae42e0 + dfe87d0 commit 4317777
Show file tree
Hide file tree
Showing 59 changed files with 1,202 additions and 332 deletions.
1 change: 1 addition & 0 deletions .github/workflows/.test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ jobs:
with:
name: test-reports-${{ env.TEST_REPORT_NAME }}
path: ./bin/testreports
retention-days: 1
-
name: Dump context
if: failure()
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/buildkit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ jobs:
name: buildkit-${{ env.PLATFORM_PAIR }}
path: ${{ env.DESTDIR }}/*
if-no-files-found: error
retention-days: 1

test:
uses: ./.github/workflows/.test.yml
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/dockerd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ jobs:
name: dockerd
path: /tmp/moby/dockerd
if-no-files-found: error
retention-days: 1

test:
runs-on: ubuntu-22.04
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/test-os.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ jobs:
with:
name: test-reports-${{ env.TEST_REPORT_NAME }}
path: ./bin/testreports
retention-days: 1
-
name: Dump context
if: failure()
Expand Down
13 changes: 9 additions & 4 deletions exporter/containerimage/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -419,11 +419,16 @@ func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCo
var divergedFromBase bool
for i, desc := range remoteDescriptors {
i, desc := i, desc
info, err := cs.Info(ctx, desc.Digest)
if err != nil {
return nil, err
// Usually we get non-empty diffID here, but if the content was ingested via a third-party containerd client,
// diffID here can be empty, and will be computed by the converter.
diffID := digest.Digest(desc.Annotations[labels.LabelUncompressed])
if diffID == "" {
info, err := cs.Info(ctx, desc.Digest)
if err != nil {
return nil, err
}
diffID = digest.Digest(info.Labels[labels.LabelUncompressed]) // can be still empty
}
diffID := digest.Digest(info.Labels[labels.LabelUncompressed]) // can be empty
var immDiffID digest.Digest
if !divergedFromBase && baseImg != nil && i < len(baseImg.RootFS.DiffIDs) {
immDiffID = baseImg.RootFS.DiffIDs[i]
Expand Down
186 changes: 113 additions & 73 deletions frontend/dockerfile/dockerfile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6856,7 +6856,16 @@ func testReproSourceDateEpoch(t *testing.T, sb integration.Sandbox) {
tm := time.Date(2023, time.January, 10, 12, 34, 56, 0, time.UTC) // 1673354096
t.Logf("SOURCE_DATE_EPOCH=%d", tm.Unix())

dockerfile := []byte(`# The base image cannot be busybox, due to https://github.com/moby/buildkit/issues/3455
type testCase struct {
name string
dockerfile string
files []fstest.Applier
expectedDigest string
}
testCases := []testCase{
{
name: "Basic",
dockerfile: `# The base image could not be busybox, due to https://github.com/moby/buildkit/issues/3455
FROM amd64/debian:bullseye-20230109-slim
RUN touch /foo
RUN touch /foo.1
Expand All @@ -6867,7 +6876,20 @@ RUN touch -d '2030-01-01 12:34:56' /foo-2030.1
RUN rm -f /foo.1
RUN rm -f /foo-2010.1
RUN rm -f /foo-2030.1
`)
`,
expectedDigest: "sha256:04e5d0cbee3317c79f50494cfeb4d8a728402a970ef32582ee47c62050037e3f",
},
{
// https://github.com/moby/buildkit/issues/4746
name: "CopyLink",
dockerfile: `FROM amd64/debian:bullseye-20230109-slim
COPY --link foo foo
`,
files: []fstest.Applier{fstest.CreateFile("foo", []byte("foo"), 0600)},
expectedDigest: "sha256:9f75e4bdbf3d825acb36bb603ddef4a25742afb8ccb674763ffc611ae047d8a6",
},
}

// https://explore.ggcr.dev/?image=amd64%2Fdebian%3Abullseye-20230109-slim
baseImageLayers := []digest.Digest{
"sha256:8740c948ffd4c816ea7ca963f99ca52f4788baa23f228da9581a9ea2edd3fcd7",
Expand All @@ -6877,88 +6899,98 @@ RUN rm -f /foo-2030.1
timeMustParse(t, time.RFC3339Nano, "2023-01-11T02:34:44.829692296Z"),
}

const expectedDigest = "sha256:04e5d0cbee3317c79f50494cfeb4d8a728402a970ef32582ee47c62050037e3f"

dir := integration.Tmpdir(
t,
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)

ctx := sb.Context()
c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()

target := registry + "/buildkit/testreprosourcedateepoch:" + fmt.Sprintf("%d", tm.Unix())
solveOpt := client.SolveOpt{
FrontendAttrs: map[string]string{
"build-arg:SOURCE_DATE_EPOCH": fmt.Sprintf("%d", tm.Unix()),
"platform": "linux/amd64",
},
LocalMounts: map[string]fsutil.FS{
dockerui.DefaultLocalNameDockerfile: dir,
dockerui.DefaultLocalNameContext: dir,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterImage,
Attrs: map[string]string{
"name": target,
"push": "true",
"oci-mediatypes": "true",
"rewrite-timestamp": "true",
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
dir := integration.Tmpdir(
t,
append([]fstest.Applier{fstest.CreateFile("Dockerfile", []byte(tc.dockerfile), 0600)}, tc.files...)...,
)

target := registry + "/buildkit/testreprosourcedateepoch-" + strings.ToLower(tc.name) + ":" + fmt.Sprintf("%d", tm.Unix())
solveOpt := client.SolveOpt{
FrontendAttrs: map[string]string{
"build-arg:SOURCE_DATE_EPOCH": fmt.Sprintf("%d", tm.Unix()),
"platform": "linux/amd64",
},
},
},
CacheExports: []client.CacheOptionsEntry{
{
Type: "registry",
Attrs: map[string]string{
"ref": target + "-cache",
"oci-mediatypes": "true",
"image-manifest": "true",
LocalMounts: map[string]fsutil.FS{
dockerui.DefaultLocalNameDockerfile: dir,
dockerui.DefaultLocalNameContext: dir,
},
},
},
}
_, err = f.Solve(ctx, c, solveOpt, nil)
require.NoError(t, err)
Exports: []client.ExportEntry{
{
Type: client.ExporterImage,
Attrs: map[string]string{
"name": target,
"push": "true",
"oci-mediatypes": "true",
"rewrite-timestamp": "true",
},
},
},
CacheExports: []client.CacheOptionsEntry{
{
Type: "registry",
Attrs: map[string]string{
"ref": target + "-cache",
"oci-mediatypes": "true",
"image-manifest": "true",
},
},
},
}
_, err = f.Solve(ctx, c, solveOpt, nil)
require.NoError(t, err)

desc, manifest, img := readImage(t, ctx, target)
_, cacheManifest, _ := readImage(t, ctx, target+"-cache")
t.Log("The digest may change depending on the BuildKit version, the snapshotter configuration, etc.")
require.Equal(t, expectedDigest, desc.Digest.String())
desc, manifest, img := readImage(t, ctx, target)
_, cacheManifest, _ := readImage(t, ctx, target+"-cache")
t.Log("The digest may change depending on the BuildKit version, the snapshotter configuration, etc.")
require.Equal(t, tc.expectedDigest, desc.Digest.String())

// Image history from the base config must remain immutable
for i, tm := range baseImageHistoryTimestamps {
require.True(t, img.History[i].Created.Equal(tm))
}
// Image history from the base config must remain immutable
for i, tm := range baseImageHistoryTimestamps {
require.True(t, img.History[i].Created.Equal(tm))
}

// Image layers, *except the base layers*, must have rewritten-timestamp
for i, l := range manifest.Layers {
if i < len(baseImageLayers) {
require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
require.Equal(t, baseImageLayers[i], l.Digest)
} else {
require.Equal(t, fmt.Sprintf("%d", tm.Unix()), l.Annotations["buildkit/rewritten-timestamp"])
}
}
// Cache layers must *not* have rewritten-timestamp
for _, l := range cacheManifest.Layers {
require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
}
// Image layers, *except the base layers*, must have rewritten-timestamp
for i, l := range manifest.Layers {
if i < len(baseImageLayers) {
require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
require.Equal(t, baseImageLayers[i], l.Digest)
} else {
require.Equal(t, fmt.Sprintf("%d", tm.Unix()), l.Annotations["buildkit/rewritten-timestamp"])
}
}
// Cache layers must *not* have rewritten-timestamp
for _, l := range cacheManifest.Layers {
require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
}

// Build again, but without rewrite-timestamp
solveOpt2 := solveOpt
delete(solveOpt2.Exports[0].Attrs, "rewrite-timestamp")
_, err = f.Solve(ctx, c, solveOpt2, nil)
require.NoError(t, err)
_, manifest2, img2 := readImage(t, ctx, target)
for i, tm := range baseImageHistoryTimestamps {
require.True(t, img2.History[i].Created.Equal(tm))
}
for _, l := range manifest2.Layers {
require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
// Build again, after pruning the base image layer cache.
// For testing https://github.com/moby/buildkit/issues/4746
ensurePruneAll(t, c, sb)
_, err = f.Solve(ctx, c, solveOpt, nil)
require.NoError(t, err)
descAfterPrune, _, _ := readImage(t, ctx, target)
require.Equal(t, desc.Digest.String(), descAfterPrune.Digest.String())

// Build again, but without rewrite-timestamp
solveOpt2 := solveOpt
delete(solveOpt2.Exports[0].Attrs, "rewrite-timestamp")
_, err = f.Solve(ctx, c, solveOpt2, nil)
require.NoError(t, err)
_, manifest2, img2 := readImage(t, ctx, target)
for i, tm := range baseImageHistoryTimestamps {
require.True(t, img2.History[i].Created.Equal(tm))
}
for _, l := range manifest2.Layers {
require.Empty(t, l.Annotations["buildkit/rewritten-timestamp"])
}
})
}
}

Expand All @@ -6978,6 +7010,14 @@ func readImage(t *testing.T, ctx context.Context, ref string) (ocispecs.Descript
require.NoError(t, json.Unmarshal(dt, &manifest))
imgDt, err := content.ReadBlob(ctx, provider, manifest.Config)
require.NoError(t, err)
// Verify that all the layer blobs are present
for _, layer := range manifest.Layers {
layerRA, err := provider.ReaderAt(ctx, layer)
require.NoError(t, err)
layerDigest, err := layer.Digest.Algorithm().FromReader(content.NewReader(layerRA))
require.NoError(t, err)
require.Equal(t, layer.Digest, layerDigest)
}
var img ocispecs.Image
require.NoError(t, json.Unmarshal(imgDt, &img))
return desc, manifest, img
Expand Down
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.21.0
go.opentelemetry.io/otel/trace v1.21.0
go.opentelemetry.io/proto/otlp v1.0.0
golang.org/x/crypto v0.17.0
golang.org/x/crypto v0.21.0
golang.org/x/mod v0.13.0
golang.org/x/net v0.18.0
golang.org/x/net v0.23.0
golang.org/x/sync v0.5.0
golang.org/x/sys v0.16.0
golang.org/x/sys v0.18.0
golang.org/x/time v0.3.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b
google.golang.org/grpc v1.59.0
Expand Down
16 changes: 8 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -473,8 +473,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
Expand All @@ -497,8 +497,8 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
Expand Down Expand Up @@ -537,11 +537,11 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand Down
10 changes: 9 additions & 1 deletion util/converter/tarconverter/tarconverter.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
type HeaderConverter func(*tar.Header)

// NewReader returns a reader that applies headerConverter.
// srcContent is drained until hitting EOF.
// Forked from https://github.com/moby/moby/blob/v24.0.6/pkg/archive/copy.go#L308-L373 .
func NewReader(srcContent io.Reader, headerConverter HeaderConverter) io.ReadCloser {
rebased, w := io.Pipe()
Expand All @@ -21,7 +22,14 @@ func NewReader(srcContent io.Reader, headerConverter HeaderConverter) io.ReadClo
if err == io.EOF {
// Signals end of archive.
rebasedTar.Close()
w.Close()
// drain the reader into io.Discard, until hitting EOF
// https://github.com/moby/buildkit/pull/4807#discussion_r1544621787
_, err = io.Copy(io.Discard, srcContent)
if err != nil {
w.CloseWithError(err)
} else {
w.Close()
}
return
}
if err != nil {
Expand Down
39 changes: 0 additions & 39 deletions vendor/golang.org/x/crypto/internal/poly1305/bits_compat.go

This file was deleted.

Loading

0 comments on commit 4317777

Please sign in to comment.