[18.09 backport] builder / buildkit updates

cli/command/cli.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"time"
 
 	"github.com/docker/cli/cli"
@@ -133,6 +134,20 @@ func (cli *DockerCli) ContentTrustEnabled() bool {
 	return cli.contentTrust
 }
 
+// BuildKitEnabled returns whether buildkit is enabled either through a daemon setting
+// or otherwise the client-side DOCKER_BUILDKIT environment variable
+func BuildKitEnabled(si ServerInfo) (bool, error) {
+	buildkitEnabled := si.BuildkitVersion == types.BuilderBuildKit
+	if buildkitEnv := os.Getenv("DOCKER_BUILDKIT"); buildkitEnv != "" {
+		var err error
+		buildkitEnabled, err = strconv.ParseBool(buildkitEnv)
+		if err != nil {
+			return false, errors.Wrap(err, "DOCKER_BUILDKIT environment variable expects boolean value")
+		}
+	}
+	return buildkitEnabled, nil
+}
+
 // ManifestStore returns a store for local manifests
 func (cli *DockerCli) ManifestStore() manifeststore.Store {
 	// TODO: support override default location from config file

cli/command/image/build.go

@@ -13,7 +13,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
-	"strconv"
 	"strings"
 
 	"github.com/docker/cli/cli"
@@ -73,6 +72,7 @@ type buildOptions struct {
 	platform       string
 	untrusted      bool
 	secrets        []string
+	ssh            []string
 }
 
 // dockerfileFromStdin returns true when the user specified that the Dockerfile
@@ -136,6 +136,8 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command {
 	flags.BoolVar(&options.pull, "pull", false, "Always attempt to pull a newer version of the image")
 	flags.StringSliceVar(&options.cacheFrom, "cache-from", []string{}, "Images to consider as cache sources")
 	flags.BoolVar(&options.compress, "compress", false, "Compress the build context using gzip")
+	flags.SetAnnotation("compress", "no-buildkit", nil)
+
 	flags.StringSliceVar(&options.securityOpt, "security-opt", []string{}, "Security options")
 	flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build")
 	flags.SetAnnotation("network", "version", []string{"1.25"})
@@ -153,11 +155,18 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command {
 	flags.BoolVar(&options.stream, "stream", false, "Stream attaches to server to negotiate build context")
 	flags.SetAnnotation("stream", "experimental", nil)
 	flags.SetAnnotation("stream", "version", []string{"1.31"})
+	flags.SetAnnotation("stream", "no-buildkit", nil)
 
-	flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (only if BuildKit enabled) (auto, plain, tty). Use plain to show container output")
+	flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output")
+	flags.SetAnnotation("progress", "buildkit", nil)
 
 	flags.StringArrayVar(&options.secrets, "secret", []string{}, "Secret file to expose to the build (only if BuildKit enabled): id=mysecret,src=/local/secret")
 	flags.SetAnnotation("secret", "version", []string{"1.39"})
+	flags.SetAnnotation("secret", "buildkit", nil)
+
+	flags.StringArrayVar(&options.ssh, "ssh", []string{}, "SSH agent socket or keys to expose to the build (only if BuildKit enabled) (format: default|<id>[=<socket>|<key>[,<key>]])")
+	flags.SetAnnotation("ssh", "version", []string{"1.39"})
+	flags.SetAnnotation("ssh", "buildkit", nil)
 	return cmd
 }
 
@@ -179,22 +188,17 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error {
 
 // nolint: gocyclo
 func runBuild(dockerCli command.Cli, options buildOptions) error {
-	if buildkitEnv := os.Getenv("DOCKER_BUILDKIT"); buildkitEnv != "" {
-		enableBuildkit, err := strconv.ParseBool(buildkitEnv)
-		if err != nil {
-			return errors.Wrap(err, "DOCKER_BUILDKIT environment variable expects boolean value")
-		}
-		if enableBuildkit {
-			return runBuildBuildKit(dockerCli, options)
-		}
-	} else if dockerCli.ServerInfo().BuildkitVersion == types.BuilderBuildKit {
+	buildkitEnabled, err := command.BuildKitEnabled(dockerCli.ServerInfo())
+	if err != nil {
+		return err
+	}
+	if buildkitEnabled {
 		return runBuildBuildKit(dockerCli, options)
 	}
 
 	var (
 		buildCtx      io.ReadCloser
 		dockerfileCtx io.ReadCloser
-		err           error
 		contextDir    string
 		tempDir       string
 		relDockerfile string

cli/command/image/build_buildkit.go

@@ -27,10 +27,11 @@ import (
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/session/secrets/secretsprovider"
+	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/progress/progressui"
 	"github.com/pkg/errors"
-	"github.com/tonistiigi/fsutil"
+	fsutiltypes "github.com/tonistiigi/fsutil/types"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -138,6 +139,13 @@ func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error {
 		}
 		s.Allow(sp)
 	}
+	if len(options.ssh) > 0 {
+		sshp, err := parseSSHSpecs(options.ssh)
+		if err != nil {
+			return errors.Wrapf(err, "could not parse ssh: %v", options.ssh)
+		}
+		s.Allow(sshp)
+	}
 
 	eg, ctx := errgroup.WithContext(ctx)
 
@@ -291,7 +299,7 @@ func doBuild(ctx context.Context, eg *errgroup.Group, dockerCli command.Cli, opt
 	return err
 }
 
-func resetUIDAndGID(s *fsutil.Stat) bool {
+func resetUIDAndGID(s *fsutiltypes.Stat) bool {
 	s.Uid = 0
 	s.Gid = 0
 	return true
@@ -408,3 +416,26 @@ func parseSecret(value string) (*secretsprovider.FileSource, error) {
 	}
 	return &fs, nil
 }
+
+func parseSSHSpecs(sl []string) (session.Attachable, error) {
+	configs := make([]sshprovider.AgentConfig, 0, len(sl))
+	for _, v := range sl {
+		c, err := parseSSH(v)
+		if err != nil {
+			return nil, err
+		}
+		configs = append(configs, *c)
+	}
+	return sshprovider.NewSSHAgentProvider(configs)
+}
+
+func parseSSH(value string) (*sshprovider.AgentConfig, error) {
+	parts := strings.SplitN(value, "=", 2)
+	cfg := sshprovider.AgentConfig{
+		ID: parts[0],
+	}
+	if len(parts) > 1 {
+		cfg.Paths = strings.Split(parts[1], ",")
+	}
+	return &cfg, nil
+}

cmd/docker/docker.go

@@ -100,8 +100,10 @@ func setHelpFunc(dockerCli *command.DockerCli, cmd *cobra.Command, flags *pflag.
 			ccmd.Println(err)
 			return
 		}
-
-		hideUnsupportedFeatures(ccmd, dockerCli)
+		if err := hideUnsupportedFeatures(ccmd, dockerCli); err != nil {
+			ccmd.Println(err)
+			return
+		}
 		defaultHelpFunc(ccmd, args)
 	})
 }
@@ -235,15 +237,21 @@ func hideFeatureSubCommand(subcmd *cobra.Command, hasFeature bool, annotation st
 	}
 }
 
-func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) {
+func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) error {
 	clientVersion := details.Client().ClientVersion()
 	osType := details.ServerInfo().OSType
 	hasExperimental := details.ServerInfo().HasExperimental
 	hasExperimentalCLI := details.ClientInfo().HasExperimental
+	hasBuildKit, err := command.BuildKitEnabled(details.ServerInfo())
+	if err != nil {
+		return err
+	}
 
 	cmd.Flags().VisitAll(func(f *pflag.Flag) {
 		hideFeatureFlag(f, hasExperimental, "experimental")
 		hideFeatureFlag(f, hasExperimentalCLI, "experimentalCLI")
+		hideFeatureFlag(f, hasBuildKit, "buildkit")
+		hideFeatureFlag(f, !hasBuildKit, "no-buildkit")
 		// hide flags not supported by the server
 		if !isOSTypeSupported(f, osType) || !isVersionSupported(f, clientVersion) {
 			f.Hidden = true
@@ -259,6 +267,8 @@ func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) {
 	for _, subcmd := range cmd.Commands() {
 		hideFeatureSubCommand(subcmd, hasExperimental, "experimental")
 		hideFeatureSubCommand(subcmd, hasExperimentalCLI, "experimentalCLI")
+		hideFeatureSubCommand(subcmd, hasBuildKit, "buildkit")
+		hideFeatureSubCommand(subcmd, !hasBuildKit, "no-buildkit")
 		// hide subcommands not supported by the server
 		if subcmdVersion, ok := subcmd.Annotations["version"]; ok && versions.LessThan(clientVersion, subcmdVersion) {
 			subcmd.Hidden = true
@@ -267,6 +277,7 @@ func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) {
 			subcmd.Hidden = true
 		}
 	}
+	return nil
 }
 
 // Checks if a command or one of its ancestors is in the list
@@ -313,6 +324,7 @@ func areFlagsSupported(cmd *cobra.Command, details versionDetails) error {
 			if _, ok := f.Annotations["experimentalCLI"]; ok && !hasExperimentalCLI {
 				errs = append(errs, fmt.Sprintf("\"--%s\" is on a Docker cli with experimental cli features enabled", f.Name))
 			}
+			// buildkit-specific flags are noop when buildkit is not enabled, so we do not add an error in that case
 		}
 	})
 	if len(errs) > 0 {

docs/reference/commandline/build.md

@@ -59,6 +59,7 @@ Options:
                                 Unit is optional and can be `b` (bytes), `k` (kilobytes), `m` (megabytes),
                                 or `g` (gigabytes). If you omit the unit, the system uses bytes.
       --squash                  Squash newly built layers into a single new layer (**Experimental Only**)
+      --ssh                     SSH agent socket or keys to expose to the build (only if BuildKit enabled) (format: default|<id>[=<socket>|<key>[,<key>]])
   -t, --tag value               Name and optionally a tag in the 'name:tag' format (default [])
       --target string           Set the target build stage to build.
       --ulimit value            Ulimit options (default [])

vendor.conf

@@ -51,7 +51,7 @@ github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55
 github.com/Microsoft/go-winio v0.4.10
 github.com/miekg/pkcs11 287d9350987cc9334667882061e202e96cdfb4d0
 github.com/mitchellh/mapstructure f15292f7a699fcc1a38a80977f80a046874ba8ac
-github.com/moby/buildkit 6812dac65e0440bb75affce1fb2175e640edc15d
+github.com/moby/buildkit 520201006c9dc676da9cf9655337ac711f7f127d
 github.com/modern-go/concurrent bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 # 1.0.3
 github.com/modern-go/reflect2 4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd # 1.0.1
 github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b
@@ -76,12 +76,12 @@ github.com/spf13/cobra v0.0.3
 github.com/spf13/pflag 4cb166e4f25ac4e8016a3595bbf7ea2e9aa85a2c https://github.com/thaJeztah/pflag.git
 github.com/syndtr/gocapability 2c00daeb6c3b45114c80ac44119e7b8801fdd852
 github.com/theupdateframework/notary v0.6.1
-github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
+github.com/tonistiigi/fsutil f567071bed2416e4d87d260d3162722651182317
 github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
 github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6
 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b
 github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d
-golang.org/x/crypto a2144134853fc9a27a7b1e3eb4f19f1a76df13c9
+golang.org/x/crypto 0709b304e793a5edb4a2c0145f281ecdc20838a4
 golang.org/x/net a680a1efc54dd51c040b3b5ce4939ea3cf2ea0d1
 golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca
 golang.org/x/sys 1b2967e3c290b7c545b3db0deeda16e9be4f98a2