Skip to content

Commit

Permalink
Merge commit '585d8c719ab77ed07380656dd95a986628bca78c' into acb/upda…
Browse files Browse the repository at this point in the history
…te-buildkit
  • Loading branch information
alexcb committed Jan 26, 2024
2 parents 086f60e + 585d8c7 commit a552655
Show file tree
Hide file tree
Showing 185 changed files with 6,539 additions and 5,532 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ BuildKit builds are based on a binary intermediate format called LLB that is use

See [`solver/pb/ops.proto`](./solver/pb/ops.proto) for the format definition, and see [`./examples/README.md`](./examples/README.md) for example LLB applications.

Currently, the following high-level languages has been implemented for LLB:
Currently, the following high-level languages have been implemented for LLB:

- Dockerfile (See [Exploring Dockerfiles](#exploring-dockerfiles))
- [Buildpacks](https://github.com/tonistiigi/buildkit-pack)
Expand Down Expand Up @@ -536,7 +536,7 @@ S3 configuration:
AWS Authentication:

The simplest way is to use an IAM Instance profile.
Others options are:
Other options are:

* Any system using environment variables / config files supported by the [AWS Go SDK](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). The configuration must be available for the buildkit daemon, not for the client.
* Using the following attributes:
Expand Down
2 changes: 1 addition & 1 deletion docker-bake.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ target "lint" {
matrix = {
buildtags = [
{ name = "default", tags = "", target = "golangci-lint" },
{ name = "labs", tags = "dfrunsecurity", target = "golangci-lint" },
{ name = "labs", tags = "dfrunsecurity dfparents", target = "golangci-lint" },
{ name = "nydus", tags = "nydus", target = "golangci-lint" },
{ name = "yaml", tags = "", target = "yamllint" },
{ name = "proto", tags = "", target = "protolint" },
Expand Down
2 changes: 1 addition & 1 deletion docs/stargz-estargz.md
Original file line number Diff line number Diff line change
Expand Up @@ -264,5 +264,5 @@ You can also create any stargz/eStargz image using the variety of tools includin
- [`ctr-remote`](https://github.com/containerd/stargz-snapshotter/blob/v0.13.0/docs/ctr-remote.md): containerd CLI developed in stargz snapshotter project. This supports converting an OCI/Docker image into eStargz and [optimizing](https://github.com/containerd/stargz-snapshotter/blob/v0.13.0/docs/estargz.md#example-use-case-of-prioritized-files-workload-based-image-optimization-in-stargz-snapshotter) it.
- [`stargzify`](https://github.com/google/crfs/tree/master/stargz/stargzify): CLI tool to convert an OCI/Docker image to stargz. This is developed in CRFS project. Creating eStargz is unsupported.

There also other tools including Kaniko, ko, builpacks.io that support eStargz creation.
There are also other tools including Kaniko, ko, builpacks.io that support eStargz creation.
For more details, please refer to [`Creating eStargz images with tools in the community` section in the introductory post](https://medium.com/nttlabs/lazy-pulling-estargz-ef35812d73de).
2 changes: 1 addition & 1 deletion executor/oci/user.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/containerd/containerd/containers"
containerdoci "github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs"
"github.com/opencontainers/runc/libcontainer/user"
"github.com/moby/sys/user"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
)
Expand Down
10 changes: 10 additions & 0 deletions frontend/dockerfile/dockerfile2llb/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -761,6 +761,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
chown: c.Chown,
chmod: c.Chmod,
link: c.Link,
parents: c.Parents,
location: c.Location(),
opt: opt,
})
Expand Down Expand Up @@ -1167,6 +1168,14 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
AllowEmptyWildcard: true,
}}, copyOpt...)

if cfg.parents {
path := strings.TrimPrefix(src, "/")
opts = append(opts, &llb.CopyInfo{
IncludePatterns: []string{path},
})
src = "/"
}

if a == nil {
a = llb.Copy(cfg.source, src, dest, opts...)
} else {
Expand Down Expand Up @@ -1257,6 +1266,7 @@ type copyConfig struct {
link bool
keepGitDir bool
checksum digest.Digest
parents bool
location []parser.Range
opt dispatchOpt
}
Expand Down
76 changes: 76 additions & 0 deletions frontend/dockerfile/dockerfile_parents_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
//go:build dfparents
// +build dfparents

package dockerfile

import (
"os"
"path/filepath"
"testing"

"github.com/containerd/continuity/fs/fstest"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/frontend/dockerui"
"github.com/moby/buildkit/util/testutil/integration"
"github.com/stretchr/testify/require"
)

var parentsTests = integration.TestFuncs(
testCopyParents,
)

func init() {
allTests = append(allTests, parentsTests...)
}

func testCopyParents(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)

dockerfile := []byte(`
FROM scratch
COPY --parents foo1/foo2/bar /
WORKDIR /test
COPY --parents foo1/foo2/ba* .
`)

dir := integration.Tmpdir(
t,
fstest.CreateFile("Dockerfile", dockerfile, 0600),
fstest.CreateDir("foo1", 0700),
fstest.CreateDir("foo1/foo2", 0700),
fstest.CreateFile("foo1/foo2/bar", []byte(`testing`), 0600),
fstest.CreateFile("foo1/foo2/baz", []byte(`testing2`), 0600),
)

c, err := client.New(sb.Context(), sb.Address())
require.NoError(t, err)
defer c.Close()

destDir := t.TempDir()

_, err = f.Solve(sb.Context(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
dockerui.DefaultLocalNameDockerfile: dir,
dockerui.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)

dt, err := os.ReadFile(filepath.Join(destDir, "foo1/foo2/bar"))
require.NoError(t, err)
require.Equal(t, "testing", string(dt))

dt, err = os.ReadFile(filepath.Join(destDir, "test/foo1/foo2/bar"))
require.NoError(t, err)
require.Equal(t, "testing", string(dt))
dt, err = os.ReadFile(filepath.Join(destDir, "test/foo1/foo2/baz"))
require.NoError(t, err)
require.Equal(t, "testing2", string(dt))
}
42 changes: 41 additions & 1 deletion frontend/dockerfile/docs/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -1457,6 +1457,46 @@ path, using `--link` is always recommended. The performance of `--link` is
equivalent or better than the default behavior and, it creates much better
conditions for cache reuse.


## COPY --parents

> **Note**
>
> Available in [`docker/dockerfile-upstream:master-labs`](#syntax).
> Will be included in `docker/dockerfile:1.6-labs`.
```dockerfile
COPY [--parents[=<boolean>]] <src>... <dest>
```

The `--parents` flag preserves parent directories for `src` entries. This flag defaults to `false`.

```dockerfile
# syntax=docker/dockerfile-upstream:master-labs
FROM scratch

COPY ./x/a.txt ./y/a.txt /no_parents/
COPY --parents ./x/a.txt ./y/a.txt /parents/

# /no_parents/a.txt
# /parents/x/a.txt
# /parents/y/a.txt
```

This behavior is analogous to the [Linux `cp` utility's](https://www.man7.org/linux/man-pages/man1/cp.1.html)
`--parents` flag.

Note that, without the `--parents` flag specified, any filename collision will
fail the Linux `cp` operation with an explicit error message
(`cp: will not overwrite just-created './x/a.txt' with './y/a.txt'`), where the
Buildkit will silently overwrite the target file at the destination.

While it is possible to preserve the directory structure for `COPY`
instructions consisting of only one `src` entry, usually it is more beneficial
to keep the layer count in the resulting image as low as possible. Therefore,
with the `--parents` flag, the Buildkit is capable of packing multiple
`COPY` instructions together, keeping the directory structure intact.

## ENTRYPOINT

ENTRYPOINT has two forms:
Expand Down Expand Up @@ -2115,7 +2155,7 @@ RUN echo "I'm building for $TARGETPLATFORM"
| `BUILDKIT_CACHE_MOUNT_NS` | String | Set optional cache ID namespace. |
| `BUILDKIT_CONTEXT_KEEP_GIT_DIR` | Bool | Trigger git context to keep the `.git` directory. |
| `BUILDKIT_INLINE_CACHE`[^2] | Bool | Inline cache metadata to image config or not. |
| `BUILDKIT_MULTI_PLATFORM` | Bool | Opt into determnistic output regardless of multi-platform output or not. |
| `BUILDKIT_MULTI_PLATFORM` | Bool | Opt into deterministic output regardless of multi-platform output or not. |
| `BUILDKIT_SANDBOX_HOSTNAME` | String | Set the hostname (default `buildkitsandbox`) |
| `BUILDKIT_SYNTAX` | String | Set frontend image |
| `SOURCE_DATE_EPOCH` | Int | Set the UNIX timestamp for created image and layers. More info from [reproducible builds](https://reproducible-builds.org/docs/source-date-epoch/). Supported since Dockerfile 1.5, BuildKit 0.11 |
Expand Down
9 changes: 5 additions & 4 deletions frontend/dockerfile/instructions/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,11 @@ func (c *AddCommand) Expand(expander SingleWordExpander) error {
type CopyCommand struct {
withNameAndCode
SourcesAndDest
From string
Chown string
Chmod string
Link bool
From string
Chown string
Chmod string
Link bool
Parents bool // parents preserves directory structure
}

func (c *CopyCommand) Expand(expander SingleWordExpander) error {
Expand Down
4 changes: 4 additions & 0 deletions frontend/dockerfile/instructions/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ type parseRequest struct {
var parseRunPreHooks []func(*RunCommand, parseRequest) error
var parseRunPostHooks []func(*RunCommand, parseRequest) error

var parentsEnabled = false

func nodeArgs(node *parser.Node) []string {
result := []string{}
for ; node.Next != nil; node = node.Next {
Expand Down Expand Up @@ -315,6 +317,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
flFrom := req.flags.AddString("from", "")
flChmod := req.flags.AddString("chmod", "")
flLink := req.flags.AddBool("link", false)
flParents := req.flags.AddBool("parents", false)
if err := req.flags.Parse(); err != nil {
return nil, err
}
Expand All @@ -331,6 +334,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
Chown: flChown.Value,
Chmod: flChmod.Value,
Link: flLink.Value == "true",
Parents: (flParents.Value == "true") && parentsEnabled, // silently ignore if not -labs
}, nil
}

Expand Down
8 changes: 8 additions & 0 deletions frontend/dockerfile/instructions/parse_parents.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
//go:build dfparents
// +build dfparents

package instructions

func init() {
parentsEnabled = true
}
2 changes: 1 addition & 1 deletion frontend/dockerfile/release/labs/tags
Original file line number Diff line number Diff line change
@@ -1 +1 @@
dfrunsecurity
dfrunsecurity dfparents
26 changes: 13 additions & 13 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
github.com/aws/smithy-go v1.13.5
github.com/containerd/console v1.0.3
github.com/containerd/containerd v1.7.7
github.com/containerd/containerd v1.7.8
github.com/containerd/continuity v0.4.2
github.com/containerd/fuse-overlayfs-snapshotter v1.0.2
github.com/containerd/go-cni v1.1.9
github.com/containerd/go-runc v1.1.0
github.com/containerd/nydus-snapshotter v0.8.2
github.com/containerd/nydus-snapshotter v0.13.1
github.com/containerd/stargz-snapshotter v0.14.3
github.com/containerd/stargz-snapshotter/estargz v0.14.3
github.com/containerd/typeurl/v2 v2.1.1
Expand Down Expand Up @@ -50,10 +50,10 @@ require (
github.com/moby/sys/mount v0.3.3
github.com/moby/sys/mountinfo v0.6.2
github.com/moby/sys/signal v0.7.0
github.com/moby/sys/user v0.1.0
github.com/morikuni/aec v1.0.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc5
github.com/opencontainers/runc v1.1.9
github.com/opencontainers/runtime-spec v1.1.0-rc.2
github.com/opencontainers/selinux v1.11.0
github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170
Expand Down Expand Up @@ -85,19 +85,16 @@ require (
go.opentelemetry.io/proto/otlp v0.19.0
golang.org/x/crypto v0.14.0
golang.org/x/net v0.17.0
golang.org/x/sync v0.1.0
golang.org/x/sync v0.3.0
golang.org/x/sys v0.13.0
golang.org/x/time v0.3.0
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
google.golang.org/grpc v1.56.3
google.golang.org/protobuf v1.30.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98
google.golang.org/grpc v1.58.3
google.golang.org/protobuf v1.31.0
kernel.org/pub/linux/libs/security/libcap/cap v1.2.67
)

require (
github.com/docker/distribution v2.8.1+incompatible
github.com/dustin/go-humanize v1.0.0
)
require github.com/docker/distribution v2.8.2+incompatible

require (
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
Expand Down Expand Up @@ -152,6 +149,7 @@ require (
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/opencontainers/runc v1.1.9 // indirect
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
Expand All @@ -164,9 +162,11 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
go.opentelemetry.io/otel/metric v0.37.0 // indirect
golang.org/x/mod v0.9.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/tools v0.10.0 // indirect
google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
kernel.org/pub/linux/libs/security/libcap/psx v1.2.67 // indirect
Expand Down
Loading

0 comments on commit a552655

Please sign in to comment.