Plugable secret backend

This commit extends SwarmKit secret management with pluggable secret
backends support.

Following previous commits:
1. moby/swarmkit@eebac27
2. moby/moby@08f7cf0

Added driver parameter to `docker secret` command.
Specifically:
1. `docker secret create [secret_name] --driver [driver_name]`
2.  Displaying the driver in
```
    $ docker secret ls
    $ docker secret inspect [secret_name]
    $ docker secret inspect [secret_name] -pretty
```

There is a bug in serialization of the secret data. Handled in
moby/moby#34157.

Signed-off-by: Liron Levin <liron@twistlock.com>

cli/command/formatter/secret.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	defaultSecretTableFormat           = "table {{.ID}}\t{{.Name}}\t{{.CreatedAt}}\t{{.UpdatedAt}}"
+	defaultSecretTableFormat           = "table {{.ID}}\t{{.Name}}\t{{.Driver}}\t{{.CreatedAt}}\t{{.UpdatedAt}}"
 	secretIDHeader                     = "ID"
 	secretCreatedHeader                = "CREATED"
 	secretUpdatedHeader                = "UPDATED"
@@ -23,6 +23,7 @@ Labels:
 {{- range $k, $v := .Labels }}
  - {{ $k }}{{if $v }}={{ $v }}{{ end }}
 {{- end }}{{ end }}
+Driver:                 {{.Driver}}
 Created at:            	{{.CreatedAt}}
 Updated at:            	{{.UpdatedAt}}`
 )
@@ -61,6 +62,7 @@ func newSecretContext() *secretContext {
 	sCtx.header = map[string]string{
 		"ID":        secretIDHeader,
 		"Name":      nameHeader,
+		"Driver":    driverHeader,
 		"CreatedAt": secretCreatedHeader,
 		"UpdatedAt": secretUpdatedHeader,
 		"Labels":    labelsHeader,
@@ -89,6 +91,13 @@ func (c *secretContext) CreatedAt() string {
 	return units.HumanDuration(time.Now().UTC().Sub(c.s.Meta.CreatedAt)) + " ago"
 }
 
+func (c *secretContext) Driver() string {
+	if c.s.Spec.Driver == nil {
+		return ""
+	}
+	return c.s.Spec.Driver.Name
+}
+
 func (c *secretContext) UpdatedAt() string {
 	return units.HumanDuration(time.Now().UTC().Sub(c.s.Meta.UpdatedAt)) + " ago"
 }
@@ -153,6 +162,13 @@ func (ctx *secretInspectContext) Labels() map[string]string {
 	return ctx.Secret.Spec.Labels
 }
 
+func (ctx *secretInspectContext) Driver() string {
+	if ctx.Secret.Spec.Driver == nil {
+		return ""
+	}
+	return ctx.Secret.Spec.Driver.Name
+}
+
 func (ctx *secretInspectContext) CreatedAt() string {
 	return command.PrettyPrint(ctx.Secret.CreatedAt)
 }

cli/command/formatter/secret_test.go

@@ -28,9 +28,9 @@ func TestSecretContextFormatWrite(t *testing.T) {
 		},
 		// Table format
 		{Context{Format: NewSecretFormat("table", false)},
-			`ID                  NAME                CREATED                  UPDATED
-1                   passwords           Less than a second ago   Less than a second ago
-2                   id_rsa              Less than a second ago   Less than a second ago
+			`ID                  NAME                DRIVER              CREATED                  UPDATED
+1                   passwords                               Less than a second ago   Less than a second ago
+2                   id_rsa                                  Less than a second ago   Less than a second ago
 `},
 		{Context{Format: NewSecretFormat("table {{.Name}}", true)},
 			`NAME

cli/command/formatter/service.go

@@ -504,6 +504,9 @@ func (c *serviceContext) Replicas() string {
 }
 
 func (c *serviceContext) Image() string {
+	if c.service.Spec.TaskTemplate.ContainerSpec == nil {
+		return ""
+	}
 	image := c.service.Spec.TaskTemplate.ContainerSpec.Image
 	if ref, err := reference.ParseNormalizedNamed(image); err == nil {
 		// update image string for display, (strips any digest)

cli/command/secret/create.go

@@ -17,6 +17,7 @@ import (
 
 type createOptions struct {
 	name   string
+	driver string
 	file   string
 	labels opts.ListOpts
 }
@@ -27,17 +28,20 @@ func newSecretCreateCommand(dockerCli command.Cli) *cobra.Command {
 	}
 
 	cmd := &cobra.Command{
-		Use:   "create [OPTIONS] SECRET file|-",
+		Use:   "create [OPTIONS] SECRET [file|-]",
 		Short: "Create a secret from a file or STDIN as content",
-		Args:  cli.ExactArgs(2),
+		Args:  cli.RequiresRangeArgs(1, 2),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			options.name = args[0]
-			options.file = args[1]
+			if len(args) == 2 {
+				options.file = args[1]
+			}
 			return runSecretCreate(dockerCli, options)
 		},
 	}
 	flags := cmd.Flags()
 	flags.VarP(&options.labels, "label", "l", "Secret labels")
+	flags.StringVar(&options.driver, "driver", "", "Secret driver")
 
 	return cmd
 }
@@ -46,28 +50,26 @@ func runSecretCreate(dockerCli command.Cli, options createOptions) error {
 	client := dockerCli.Client()
 	ctx := context.Background()
 
-	var in io.Reader = dockerCli.In()
-	if options.file != "-" {
-		file, err := system.OpenSequential(options.file)
-		if err != nil {
-			return err
-		}
-		in = file
-		defer file.Close()
+	if options.driver != "" && options.file != "" {
+		return errors.Errorf("When using secret driver secret data must be empty")
 	}
 
-	secretData, err := ioutil.ReadAll(in)
+	secretData, err := readSecretData(dockerCli.In(), options.file)
 	if err != nil {
-		return errors.Errorf("Error reading content from %q: %v", options.file, err)
+		return err
 	}
-
 	spec := swarm.SecretSpec{
 		Annotations: swarm.Annotations{
 			Name:   options.name,
 			Labels: opts.ConvertKVStringsToMap(options.labels.GetAll()),
 		},
 		Data: secretData,
 	}
+	if options.driver != "" {
+		spec.Driver = &swarm.Driver{
+			Name: options.driver,
+		}
+	}
 
 	r, err := client.SecretCreate(ctx, spec)
 	if err != nil {
@@ -77,3 +79,24 @@ func runSecretCreate(dockerCli command.Cli, options createOptions) error {
 	fmt.Fprintln(dockerCli.Out(), r.ID)
 	return nil
 }
+
+func readSecretData(in io.ReadCloser, file string) ([]byte, error) {
+	// Read secret value from external driver
+	if file == "" {
+		return nil, nil
+	}
+	if file != "-" {
+		var err error
+		in, err = system.OpenSequential(file)
+		if err != nil {
+			return nil, err
+		}
+		defer in.Close()
+	}
+	var err error
+	data, err := ioutil.ReadAll(in)
+	if err != nil {
+		return nil, errors.Errorf("Error reading content from %q: %v", file, err)
+	}
+	return data, nil
+}

cli/command/secret/create_test.go

@@ -25,12 +25,8 @@ func TestSecretCreateErrors(t *testing.T) {
 		secretCreateFunc func(swarm.SecretSpec) (types.SecretCreateResponse, error)
 		expectedError    string
 	}{
-		{
-			args:          []string{"too_few"},
-			expectedError: "requires exactly 2 argument(s)",
-		},
 		{args: []string{"too", "many", "arguments"},
-			expectedError: "requires exactly 2 argument(s)",
+			expectedError: "requires at least 1 and at most 2",
 		},
 		{
 			args: []string{"name", filepath.Join("testdata", secretDataFile)},

cli/command/secret/inspect_test.go

@@ -164,6 +164,7 @@ func TestSecretInspectPretty(t *testing.T) {
 					}),
 					SecretID("secretID"),
 					SecretName("secretName"),
+					SecretDriver("driver"),
 					SecretCreatedAt(time.Time{}),
 					SecretUpdatedAt(time.Time{}),
 				), []byte{}, nil

cli/command/secret/ls_test.go

@@ -64,6 +64,7 @@ func TestSecretList(t *testing.T) {
 					SecretVersion(swarm.Version{Index: 11}),
 					SecretCreatedAt(time.Now().Add(-2*time.Hour)),
 					SecretUpdatedAt(time.Now().Add(-1*time.Hour)),
+					SecretDriver("driver"),
 				),
 			}, nil
 		},

cli/command/secret/testdata/secret-inspect-pretty.simple.golden

@@ -2,5 +2,6 @@ ID: secretID
 Name: secretName
 Labels:
 	- lbl1=value1
+Driver: driver
 Created at: 0001-01-01 00:00:00+0000 utc
 Updated at: 0001-01-01 00:00:00+0000 utc

cli/command/secret/testdata/secret-list-with-filter.golden

@@ -1,3 +1,3 @@
-ID                  NAME                CREATED             UPDATED
-ID-foo              foo                 2 hours ago         About an hour ago
-ID-bar              bar                 2 hours ago         About an hour ago
+ID                  NAME                 DRIVER                CREATED             UPDATED
+ID-foo              foo                                        2 hours ago         About an hour ago
+ID-bar              bar                                        2 hours ago         About an hour ago

cli/command/secret/testdata/secret-list.golden

@@ -1,3 +1,3 @@
-ID                  NAME                CREATED             UPDATED
-ID-foo              foo                 2 hours ago         About an hour ago
-ID-bar              bar                 2 hours ago         About an hour ago
+ID                  NAME                 DRIVER           CREATED             UPDATED
+ID-foo              foo                                  2 hours ago         About an hour ago
+ID-bar              bar                  driver          2 hours ago         About an hour ago

cli/command/service/inspect_test.go

@@ -41,7 +41,7 @@ func formatServiceInspect(t *testing.T, format formatter.Format, now time.Time)
 				Labels: map[string]string{"com.label": "foo"},
 			},
 			TaskTemplate: swarm.TaskSpec{
-				ContainerSpec: swarm.ContainerSpec{
+				ContainerSpec: &swarm.ContainerSpec{
 					Image: "foo/bar@sha256:this_is_a_test",
 				},
 				Networks: []swarm.NetworkAttachmentConfig{

cli/command/service/opts.go

@@ -592,7 +592,7 @@ func (options *serviceOptions) ToService(ctx context.Context, apiClient client.N
 			Labels: opts.ConvertKVStringsToMap(options.labels.GetAll()),
 		},
 		TaskTemplate: swarm.TaskSpec{
-			ContainerSpec: swarm.ContainerSpec{
+			ContainerSpec: &swarm.ContainerSpec{
 				Image:      options.image,
 				Args:       options.args,
 				Command:    options.entrypoint.Value(),

cli/command/service/update.go

@@ -270,7 +270,7 @@ func updateService(ctx context.Context, apiClient client.NetworkAPIClient, flags
 		}
 	}
 
-	cspec := &spec.TaskTemplate.ContainerSpec
+	cspec := spec.TaskTemplate.ContainerSpec
 	task := &spec.TaskTemplate
 
 	taskResources := func() *swarm.ResourceRequirements {

cli/command/service/update_test.go

@@ -19,8 +19,8 @@ func TestUpdateServiceArgs(t *testing.T) {
 	flags := newUpdateCommand(nil).Flags()
 	flags.Set("args", "the \"new args\"")
 
-	spec := &swarm.ServiceSpec{}
-	cspec := &spec.TaskTemplate.ContainerSpec
+	spec := &swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{}}}
+	cspec := spec.TaskTemplate.ContainerSpec
 	cspec.Args = []string{"old", "args"}
 
 	updateService(nil, nil, flags, spec)
@@ -452,8 +452,8 @@ func TestUpdateSecretUpdateInPlace(t *testing.T) {
 }
 
 func TestUpdateReadOnly(t *testing.T) {
-	spec := &swarm.ServiceSpec{}
-	cspec := &spec.TaskTemplate.ContainerSpec
+	spec := &swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{}}}
+	cspec := spec.TaskTemplate.ContainerSpec
 
 	// Update with --read-only=true, changed to true
 	flags := newUpdateCommand(nil).Flags()
@@ -474,8 +474,8 @@ func TestUpdateReadOnly(t *testing.T) {
 }
 
 func TestUpdateStopSignal(t *testing.T) {
-	spec := &swarm.ServiceSpec{}
-	cspec := &spec.TaskTemplate.ContainerSpec
+	spec := &swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{}}}
+	cspec := spec.TaskTemplate.ContainerSpec
 
 	// Update with --stop-signal=SIGUSR1
 	flags := newUpdateCommand(nil).Flags()

cli/command/stack/deploy_bundlefile.go

@@ -64,7 +64,7 @@ func deployBundle(ctx context.Context, dockerCli command.Cli, opts deployOptions
 				Labels: convert.AddStackLabel(namespace, service.Labels),
 			},
 			TaskTemplate: swarm.TaskSpec{
-				ContainerSpec: swarm.ContainerSpec{
+				ContainerSpec: &swarm.ContainerSpec{
 					Image:   service.Image,
 					Command: service.Command,
 					Args:    service.Args,

cli/compose/convert/service.go

@@ -128,7 +128,7 @@ func Service(
 			Labels: AddStackLabel(namespace, service.Deploy.Labels),
 		},
 		TaskTemplate: swarm.TaskSpec{
-			ContainerSpec: swarm.ContainerSpec{
+			ContainerSpec: &swarm.ContainerSpec{
 				Image:           service.Image,
 				Command:         service.Entrypoint,
 				Args:            service.Command,

cli/internal/test/builders/secret.go

@@ -32,6 +32,15 @@ func SecretName(name string) func(secret *swarm.Secret) {
 	}
 }
 
+// SecretDriver sets the secret's driver name
+func SecretDriver(driver string) func(secret *swarm.Secret) {
+	return func(secret *swarm.Secret) {
+		secret.Spec.Driver = &swarm.Driver{
+			Name: driver,
+		}
+	}
+}
+
 // SecretID sets the secret's ID
 func SecretID(ID string) func(secret *swarm.Secret) {
 	return func(secret *swarm.Secret) {

cli/internal/test/builders/service.go

@@ -56,7 +56,7 @@ func ReplicatedService(replicas uint64) func(*swarm.Service) {
 // ServiceImage sets the service's image
 func ServiceImage(image string) func(*swarm.Service) {
 	return func(service *swarm.Service) {
-		service.Spec.TaskTemplate = swarm.TaskSpec{ContainerSpec: swarm.ContainerSpec{Image: image}}
+		service.Spec.TaskTemplate = swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{Image: image}}
 	}
 }
 

cli/internal/test/builders/task.go

@@ -140,7 +140,7 @@ func WithTaskSpec(specBuilders ...func(*swarm.TaskSpec)) func(*swarm.Task) {
 // Any number of taskSpec function builder can be pass to augment it.
 func TaskSpec(specBuilders ...func(*swarm.TaskSpec)) *swarm.TaskSpec {
 	taskSpec := &swarm.TaskSpec{
-		ContainerSpec: swarm.ContainerSpec{
+		ContainerSpec: &swarm.ContainerSpec{
 			Image: "myimage:mytag",
 		},
 	}

vendor.conf

@@ -7,7 +7,7 @@ github.com/coreos/etcd 824277cb3a577a0e8c829ca9ec557b973fe06d20
 github.com/cpuguy83/go-md2man a65d4d2de4d5f7c74868dfa9b202a3c8be315aaa
 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76
 github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621
-github.com/docker/docker 87df0e533b619c088091fd1e2310e92bb9a24822
+github.com/docker/docker 0304c98d85404fe75a1b4a35d3c111931e062f41
 github.com/docker/docker-credential-helpers v0.5.1
 
 # the docker/go package contains a customized version of canonical/json
@@ -44,7 +44,7 @@ github.com/xeipuuv/gojsonreference e02fc20de94c78484cd5ffb007f8af96be030a45
 github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d
 golang.org/x/crypto 3fbbcd23f1cb824e69491a5930cfeff09b12f4d2
 golang.org/x/net c427ad74c6d7a814201695e9ffde0c5d400a7674
-golang.org/x/sys 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9
+golang.org/x/sys 739734461d1c916b6c72a63d7efda2b27edb369f
 golang.org/x/text f72d8390a633d5dfb0cc84043294db9f6c935756
 golang.org/x/time a4bde12657593d5e90d0533a3e4fd95e635124cb
 google.golang.org/grpc v1.0.4