Add option to push missing images (cnabio#76)

Add option to push missing images

Gopkg.toml

@@ -30,28 +30,19 @@
 
 [[constraint]]
   name = "github.com/deislabs/cnab-go"
-  revision = "v0.4.0-beta1"
+  version = "v0.7.1-beta1"
 
-# Use master while we wait for https://github.com/containerd/containerd/pull/3218 to be included in the release
 [[constraint]]
   name = "github.com/containerd/containerd"
-  branch = "master"
-
-[[override]]
-  name = "github.com/docker/cli"
-  revision = "f95ca8e1ba6c22c9abcdbf65e8dcc39c53958bba"
+  version = "v1.3.0"
 
 [[constraint]]
   name = "github.com/docker/distribution"
-  revision = "83389a148052d74ac602f5f1d62f86ff2f3c4aa5"
-
-[[override]]
-  name = "github.com/docker/docker"
-  revision = "6e3113f700dea1bf2785d94731b4b5a1e602d9ab"
+  version = "v2.7.1"
 
 [[override]]
   name = "github.com/docker/docker-credential-helpers"
-  revision = "5241b46610f2491efdf9d1c85f1ddf5b02f6d962"
+  version = "v0.6.3"
 
 [prune]
   go-tests = true

cmd/cnab-to-oci/push.go

@@ -6,10 +6,12 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"os"
 
 	"github.com/deislabs/cnab-go/bundle"
 	"github.com/docker/cnab-to-oci/remotes"
 	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/client"
 	"github.com/spf13/cobra"
 )
 
@@ -21,6 +23,7 @@ type pushOptions struct {
 	invocationPlatforms []string
 	componentPlatforms  []string
 	autoUpdateBundle    bool
+	pushImages          bool
 }
 
 func pushCmd() *cobra.Command {
@@ -44,6 +47,7 @@ func pushCmd() *cobra.Command {
 	cmd.Flags().StringSliceVar(&opts.invocationPlatforms, "invocation-platforms", nil, "Platforms to push (for multi-arch invocation images)")
 	cmd.Flags().StringSliceVar(&opts.componentPlatforms, "component-platforms", nil, "Platforms to push (for multi-arch component images)")
 	cmd.Flags().BoolVar(&opts.autoUpdateBundle, "auto-update-bundle", false, "Updates the bundle image properties with the one resolved on the registry")
+	cmd.Flags().BoolVar(&opts.pushImages, "push-images", true, "Allow to push missing images in the registry that are available in the local docker daemon image store")
 
 	return cmd
 }
@@ -71,6 +75,13 @@ func runPush(opts pushOptions) error {
 	if opts.autoUpdateBundle {
 		fixupOptions = append(fixupOptions, remotes.WithAutoBundleUpdate())
 	}
+	if opts.pushImages {
+		cli, err := client.NewClientWithOpts(client.FromEnv)
+		if err != nil {
+			return err
+		}
+		fixupOptions = append(fixupOptions, remotes.WithPushImages(cli, os.Stdout))
+	}
 	relocationMap, err := remotes.FixupBundle(context.Background(), &b, ref, resolver, fixupOptions...)
 	if err != nil {
 		return err

examples/helloworld-cnab/bundle.json

@@ -25,4 +25,4 @@
   "images": null,
   "parameters": null,
   "credentials": null
-}
+}

remotes/fixup.go

@@ -70,7 +70,7 @@ func fixupImage(ctx context.Context, baseImage *bundle.BaseImage, relocationMap
 
 	notifyEvent(FixupEventTypeCopyImageStart, "", nil)
 	// Fixup Base image
-	fixupInfo, err := fixupBaseImage(ctx, baseImage, cfg.targetRef, cfg.resolver)
+	fixupInfo, err := fixupBaseImage(ctx, baseImage, cfg)
 	if err != nil {
 		return notifyError(notifyEvent, err)
 	}
@@ -181,33 +181,66 @@ func fixupPlatforms(ctx context.Context,
 	return nil
 }
 
-func fixupBaseImage(ctx context.Context,
-	baseImage *bundle.BaseImage,
-	targetRef reference.Named, //nolint: interfacer
-	resolver remotes.Resolver) (imageFixupInfo, error) {
-
+func fixupBaseImage(ctx context.Context, baseImage *bundle.BaseImage, cfg fixupConfig) (imageFixupInfo, error) {
 	// Check image references
 	if err := checkBaseImage(baseImage); err != nil {
 		return imageFixupInfo{}, fmt.Errorf("invalid image %q: %s", baseImage.Image, err)
 	}
-	targetRepoOnly, err := reference.ParseNormalizedNamed(targetRef.Name())
+	targetRepoOnly, err := reference.ParseNormalizedNamed(cfg.targetRef.Name())
 	if err != nil {
 		return imageFixupInfo{}, err
 	}
 	sourceImageRef, err := reference.ParseNormalizedNamed(baseImage.Image)
 	if err != nil {
-		return imageFixupInfo{}, fmt.Errorf("%q is not a valid image reference for %q: %s", baseImage.Image, targetRef, err)
+		return imageFixupInfo{}, fmt.Errorf("%q is not a valid image reference for %q: %s", baseImage.Image, cfg.targetRef, err)
 	}
 	sourceImageRef = reference.TagNameOnly(sourceImageRef)
 
 	// Try to fetch the image descriptor
-	_, descriptor, err := resolver.Resolve(ctx, sourceImageRef.String())
+	_, descriptor, err := cfg.resolver.Resolve(ctx, sourceImageRef.String())
 	if err != nil {
-		return imageFixupInfo{}, fmt.Errorf("failed to resolve %q, push the image to the registry before pushing the bundle: %s", sourceImageRef, err)
+		if cfg.pushImages {
+			descriptor, err = pushImageToTarget(ctx, baseImage, cfg)
+			if err != nil {
+				return imageFixupInfo{}, err
+			}
+		} else {
+			return imageFixupInfo{}, fmt.Errorf("failed to resolve %q, push the image to the registry before pushing the bundle: %s", sourceImageRef, err)
+		}
 	}
 	return imageFixupInfo{
 		resolvedDescriptor: descriptor,
 		sourceRef:          sourceImageRef,
 		targetRepo:         targetRepoOnly,
 	}, nil
 }
+
+// pushImageToTarget pushes the image from the local docker daemon store to the target defined in the configuration.
+// Docker image cannot be pushed by digest to a registry. So to be able to push the image inside the targeted repository
+// the same behaviour than for multi architecture images is used: all the images are tagged for the targeted repository
+// and then pushed.
+// Every time a new image is pushed under a tag, the previous tagged image will be untagged. But this untagged image
+// remains accessible using its digest. So right after pushing it, the image is resolved to grab its digest from the
+// registry and can be added to the index.
+// The final workflow is then:
+//  - tag the image to push with targeted reference
+//  - push the image using a docker `ImageAPIClient`
+//  - resolve the pushed image to grab its digest
+func pushImageToTarget(ctx context.Context, baseImage *bundle.BaseImage, cfg fixupConfig) (ocischemav1.Descriptor, error) {
+	taggedRef := reference.TagNameOnly(cfg.targetRef)
+
+	if err := cfg.imageClient.ImageTag(ctx, baseImage.Image, cfg.targetRef.String()); err != nil {
+		return ocischemav1.Descriptor{}, fmt.Errorf("failed to push image %q, make sure the image exists locally: %s", baseImage.Image, err)
+	}
+
+	if err := pushTaggedImage(ctx, cfg.imageClient, cfg.targetRef, cfg.pushOut); err != nil {
+		return ocischemav1.Descriptor{}, fmt.Errorf("failed to push image %q: %s", baseImage.Image, err)
+	}
+
+	_, descriptor, err := cfg.resolver.Resolve(ctx, taggedRef.String())
+	if err != nil {
+		return ocischemav1.Descriptor{}, fmt.Errorf("failed to resolve %q after pushing it: %s", taggedRef, err)
+	}
+
+	return descriptor, nil
+}

remotes/fixupoptions.go

@@ -1,10 +1,15 @@
 package remotes
 
 import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
 	"github.com/deislabs/cnab-go/bundle"
 	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/client"
 	ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -26,6 +31,9 @@ type fixupConfig struct {
 	invocationImagePlatformFilter platforms.Matcher
 	componentImagePlatformFilter  platforms.Matcher
 	autoBundleUpdate              bool
+	pushImages                    bool
+	imageClient                   client.ImageAPIClient
+	pushOut                       io.Writer
 }
 
 // FixupOption is a helper for configuring a FixupBundle
@@ -114,3 +122,25 @@ func WithAutoBundleUpdate() FixupOption {
 		return nil
 	}
 }
+
+// WithPushImages authorizes the fixup command to push missing images.
+// By default the fixup will look at images defined in the bundle.
+// Existing images in the target registry or accessible from an other registry will be copied or mounted under the
+// target tag.
+// But local only images (for example after a local build of components of the bundle) must be pushed.
+// This option will allow to push images that are only available in the docker daemon image store to the defined target.
+func WithPushImages(imageClient client.ImageAPIClient, out io.Writer) FixupOption {
+	return func(cfg *fixupConfig) error {
+		cfg.pushImages = true
+		if imageClient == nil {
+			return fmt.Errorf("could not configure fixup, 'imageClient' cannot be nil to push images")
+		}
+		cfg.imageClient = imageClient
+		if out == nil {
+			cfg.pushOut = ioutil.Discard
+		} else {
+			cfg.pushOut = out
+		}
+		return nil
+	}
+}