On Dockerfiles, allow filepaths to be excluded from ADD and COPY commands

frontend/dockerfile/dockerfile2llb/convert.go

@@ -714,17 +714,18 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 		}
 		if err == nil {
 			err = dispatchCopy(d, copyConfig{
-				params:       c.SourcesAndDest,
-				source:       opt.buildContext,
-				isAddCommand: true,
-				cmdToPrint:   c,
-				chown:        c.Chown,
-				chmod:        c.Chmod,
-				link:         c.Link,
-				keepGitDir:   c.KeepGitDir,
-				checksum:     checksum,
-				location:     c.Location(),
-				opt:          opt,
+				params:          c.SourcesAndDest,
+				excludePatterns: c.ExcludePatterns,
+				source:          opt.buildContext,
+				isAddCommand:    true,
+				cmdToPrint:      c,
+				chown:           c.Chown,
+				chmod:           c.Chmod,
+				link:            c.Link,
+				keepGitDir:      c.KeepGitDir,
+				checksum:        checksum,
+				location:        c.Location(),
+				opt:             opt,
 			})
 		}
 		if err == nil {
@@ -762,16 +763,17 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 			l = cmd.sources[0].state
 		}
 		err = dispatchCopy(d, copyConfig{
-			params:       c.SourcesAndDest,
-			source:       l,
-			isAddCommand: false,
-			cmdToPrint:   c,
-			chown:        c.Chown,
-			chmod:        c.Chmod,
-			link:         c.Link,
-			parents:      c.Parents,
-			location:     c.Location(),
-			opt:          opt,
+			params:          c.SourcesAndDest,
+			excludePatterns: c.ExcludePatterns,
+			source:          l,
+			isAddCommand:    false,
+			cmdToPrint:      c,
+			chown:           c.Chown,
+			chmod:           c.Chmod,
+			link:            c.Link,
+			parents:         c.Parents,
+			location:        c.Location(),
+			opt:             opt,
 		})
 		if err == nil && len(cmd.sources) == 0 {
 			for _, src := range c.SourcePaths {
@@ -1058,6 +1060,14 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo
 	return nil
 }
 
+type excludeOnCopy struct {
+	patterns []string
+}
+
+func (e *excludeOnCopy) SetCopyOption(i *llb.CopyInfo) {
+	i.ExcludePatterns = append(i.ExcludePatterns, e.patterns...)
+}
+
 func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 	dest, err := pathRelativeToWorkingDir(d.state, cfg.params.DestPath, *d.platform)
 	if err != nil {
@@ -1074,6 +1084,8 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 		copyOpt = append(copyOpt, llb.WithUser(cfg.chown))
 	}
 
+	copyOpt = append(copyOpt, &excludeOnCopy{cfg.excludePatterns})
+
 	var mode *os.FileMode
 	if cfg.chmod != "" {
 		p, err := strconv.ParseUint(cfg.chmod, 8, 32)
@@ -1265,18 +1277,19 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 }
 
 type copyConfig struct {
-	params       instructions.SourcesAndDest
-	source       llb.State
-	isAddCommand bool
-	cmdToPrint   fmt.Stringer
-	chown        string
-	chmod        string
-	link         bool
-	keepGitDir   bool
-	checksum     digest.Digest
-	parents      bool
-	location     []parser.Range
-	opt          dispatchOpt
+	params          instructions.SourcesAndDest
+	excludePatterns []string
+	source          llb.State
+	isAddCommand    bool
+	cmdToPrint      fmt.Stringer
+	chown           string
+	chmod           string
+	link            bool
+	keepGitDir      bool
+	checksum        digest.Digest
+	parents         bool
+	location        []parser.Range
+	opt             dispatchOpt
 }
 
 func dispatchMaintainer(d *dispatchState, c *instructions.MaintainerCommand) error {

frontend/dockerfile/dockerfile2llb/convert_test.go

@@ -39,7 +39,7 @@ RUN ls -l
 	df = `FROM scratch AS foo
 ENV FOO bar
 FROM foo
-COPY --from=foo f1 /
+COPY --from=foo --exclude='.config' --exclude='src/*.py' f1 /
 COPY --from=0 f2 /
 	`
 	_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
@@ -208,3 +208,14 @@ COPY --from=stage1 f2 /sub/
 	_, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
 	assert.EqualError(t, err, "circular dependency detected on stage: stage0")
 }
+
+func TestDockerfileCopyExcludePatterns(t *testing.T) {
+	df := `FROM scratch
+COPY --exclude=src/*.go --exclude=tmp/*.txt dir /sub/
+`
+	state, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
+	assert.NoError(t, err)
+
+	_, err = state.Marshal(context.TODO())
+	assert.NoError(t, err)
+}

frontend/dockerfile/docs/reference.md

@@ -1059,8 +1059,8 @@ RUN apt-get update && apt-get install -y ...
 ADD has two forms:
 
 ```dockerfile
-ADD [--chown=<user>:<group>] [--chmod=<perms>] [--checksum=<checksum>] <src>... <dest>
-ADD [--chown=<user>:<group>] [--chmod=<perms>] ["<src>",... "<dest>"]
+ADD [--chown=<user>:<group>] [--chmod=<perms> [--exclude=<exclude>]... [--checksum=<checksum>] <src>... <dest>
+ADD [--chown=<user>:<group>] [--chmod=<perms> [--exclude=<exclude>]... ["<src>",... "<dest>"]
 ```
 
 The latter form is required for paths containing whitespace.
@@ -1079,20 +1079,25 @@ The latter form is required for paths containing whitespace.
 > Only octal notation is currently supported. Non-octal support is tracked in
 > [moby/buildkit#1951](https://github.com/moby/buildkit/issues/1951).
 
+> **Note**
+>
+> The `--exclude` option can be specified multiple times and cause files matching its patterns not to be copied,
+> even if the files paths match the pattern specified in `<src>`.
+
 The `ADD` instruction copies new files, directories or remote file URLs from `<src>`
 and adds them to the filesystem of the image at the path `<dest>`.
 
 Multiple `<src>` resources may be specified but if they are files or
 directories, their paths are interpreted as relative to the source of
 the context of the build.
 
-Each `<src>` may contain wildcards and matching will be done using Go's
+Each `<src>` and `<exclude>` may contain wildcards and matching will be done using Go's
 [filepath.Match](https://golang.org/pkg/path/filepath#Match) rules. For example:
 
-To add all files starting with "hom":
+To add all files starting with "hom", excluding all files with `txt` and `.md` extensions:
 
 ```dockerfile
-ADD hom* /mydir/
+ADD --exclude=*.txt --exclude=*.md hom* /mydir/
 ```
 
 In the example below, `?` is replaced with any single character, e.g., "home.txt".
@@ -1291,8 +1296,8 @@ See [`COPY --link`](#copy---link).
 COPY has two forms:
 
 ```dockerfile
-COPY [--chown=<user>:<group>] [--chmod=<perms>] <src>... <dest>
-COPY [--chown=<user>:<group>] [--chmod=<perms>] ["<src>",... "<dest>"]
+COPY [--chown=<user>:<group>] [--chmod=<perms>] [--exclude=<exclude>]... <src>... <dest>
+COPY [--chown=<user>:<group>] [--chmod=<perms>] [--exclude=<exclude>]... ["<src>",... "<dest>"]
 ```
 
 This latter form is required for paths containing whitespace
@@ -1305,6 +1310,11 @@ This latter form is required for paths containing whitespace
 > translating user and group names to IDs restricts this feature to only be viable for
 > Linux OS-based containers.
 
+> **Note**
+>
+> The `--exclude` option can be specified multiple times and cause files matching its patterns not to be copied,
+> even if the files paths match the pattern specified in `<src>`.
+
 The `COPY` instruction copies new files or directories from `<src>`
 and adds them to the filesystem of the container at the path `<dest>`.
 
@@ -1315,10 +1325,10 @@ of the build.
 Each `<src>` may contain wildcards and matching will be done using Go's
 [filepath.Match](https://golang.org/pkg/path/filepath#Match) rules. For example:
 
-To add all files starting with "hom":
+To add all files starting with "hom", excluding all files with `txt` and `.md` extensions:
 
 ```dockerfile
-COPY hom* /mydir/
+COPY  --exclude=*.txt --exclude=*.md hom* /mydir/
 ```
 
 In the example below, `?` is replaced with any single character, e.g., "home.txt".

frontend/dockerfile/instructions/commands.go

@@ -239,11 +239,12 @@ func (s *SourcesAndDest) ExpandRaw(expander SingleWordExpander) error {
 type AddCommand struct {
 	withNameAndCode
 	SourcesAndDest
-	Chown      string
-	Chmod      string
-	Link       bool
-	KeepGitDir bool // whether to keep .git dir, only meaningful for git sources
-	Checksum   string
+	Chown           string
+	Chmod           string
+	Link            bool
+	ExcludePatterns []string
+	KeepGitDir      bool // whether to keep .git dir, only meaningful for git sources
+	Checksum        string
 }
 
 func (c *AddCommand) Expand(expander SingleWordExpander) error {
@@ -270,11 +271,12 @@ func (c *AddCommand) Expand(expander SingleWordExpander) error {
 type CopyCommand struct {
 	withNameAndCode
 	SourcesAndDest
-	From    string
-	Chown   string
-	Chmod   string
-	Link    bool
-	Parents bool // parents preserves directory structure
+	From            string
+	Chown           string
+	Chmod           string
+	Link            bool
+	ExcludePatterns []string
+	Parents         bool // parents preserves directory structure
 }
 
 func (c *CopyCommand) Expand(expander SingleWordExpander) error {

frontend/dockerfile/instructions/parse.go

@@ -284,6 +284,8 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 	if len(req.args) < 2 {
 		return nil, errNoDestinationArgument("ADD")
 	}
+
+	flExcludes := req.flags.AddStrings("exclude")
 	flChown := req.flags.AddString("chown", "")
 	flChmod := req.flags.AddString("chmod", "")
 	flLink := req.flags.AddBool("link", false)
@@ -306,13 +308,16 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 		Link:            flLink.Value == "true",
 		KeepGitDir:      flKeepGitDir.Value == "true",
 		Checksum:        flChecksum.Value,
+		ExcludePatterns: flExcludes.StringValues,
 	}, nil
 }
 
 func parseCopy(req parseRequest) (*CopyCommand, error) {
 	if len(req.args) < 2 {
 		return nil, errNoDestinationArgument("COPY")
 	}
+
+	flExcludes := req.flags.AddStrings("exclude")
 	flChown := req.flags.AddString("chown", "")
 	flFrom := req.flags.AddString("from", "")
 	flChmod := req.flags.AddString("chmod", "")
@@ -335,6 +340,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
 		Chmod:           flChmod.Value,
 		Link:            flLink.Value == "true",
 		Parents:         (flParents.Value == "true") && parentsEnabled, // silently ignore if not -labs
+		ExcludePatterns: flExcludes.StringValues,
 	}, nil
 }