WIP: implement enhancements/oc/mirroring-manifests

This PR implements the current baseline of openshift/enhancements#238 (which is a continuation of the work originated here: openshift/enhancements#188)

Local package pkg/verify was reused with minor changes from CVO so there is PR openshift/library-go#725 to move package verify into library-go.

contrib/completions/bash/oc

@@ -5592,6 +5592,11 @@ _oc_adm_release_mirror()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--apply-manifests")
+    local_nonpersistent_flags+=("--apply-manifests")
+    flags+=("--config-to-dir=")
+    two_word_flags+=("--config-to-dir")
+    local_nonpersistent_flags+=("--config-to-dir=")
     flags+=("--dry-run")
     local_nonpersistent_flags+=("--dry-run")
     flags+=("--from=")
@@ -5605,6 +5610,8 @@ _oc_adm_release_mirror()
     flags+=("--max-per-registry=")
     two_word_flags+=("--max-per-registry")
     local_nonpersistent_flags+=("--max-per-registry=")
+    flags+=("--overwrite")
+    local_nonpersistent_flags+=("--overwrite")
     flags+=("--registry-config=")
     two_word_flags+=("--registry-config")
     two_word_flags+=("-a")

contrib/completions/zsh/oc

@@ -5734,6 +5734,11 @@ _oc_adm_release_mirror()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--apply-manifests")
+    local_nonpersistent_flags+=("--apply-manifests")
+    flags+=("--config-to-dir=")
+    two_word_flags+=("--config-to-dir")
+    local_nonpersistent_flags+=("--config-to-dir=")
     flags+=("--dry-run")
     local_nonpersistent_flags+=("--dry-run")
     flags+=("--from=")
@@ -5747,6 +5752,8 @@ _oc_adm_release_mirror()
     flags+=("--max-per-registry=")
     two_word_flags+=("--max-per-registry")
     local_nonpersistent_flags+=("--max-per-registry=")
+    flags+=("--overwrite")
+    local_nonpersistent_flags+=("--overwrite")
     flags+=("--registry-config=")
     two_word_flags+=("--registry-config")
     two_word_flags+=("-a")

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/openshift/api v0.0.0-20200330134433-8e259f67fc55
 	github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160
 	github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0
-	github.com/openshift/library-go v0.0.0-20200327125526-163b2f0d6264
+	github.com/openshift/library-go v0.0.0-20200402123743-4015ba624cae
 	github.com/operator-framework/operator-registry v1.5.11
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v1.1.0

go.sum

@@ -658,8 +658,8 @@ github.com/openshift/kubernetes-client-go v0.0.0-20200312102710-a3d38b2f4366 h1:
 github.com/openshift/kubernetes-client-go v0.0.0-20200312102710-a3d38b2f4366/go.mod h1:UvuVxHjKWIcgy0iMvF+bwNDW7l0mskTNOaOW1Qv5BMA=
 github.com/openshift/kubernetes-kubectl v0.0.0-20200318132203-d76c0712736d h1:/ljjf5pqefwDv0KY719ixfAoRUC0SZBHtxDSkxn+ofY=
 github.com/openshift/kubernetes-kubectl v0.0.0-20200318132203-d76c0712736d/go.mod h1:W/RhsZUzxhIrbUQLQCCKNKZkz4LVbglYCjaR8jP/JIw=
-github.com/openshift/library-go v0.0.0-20200327125526-163b2f0d6264 h1:VcZjTupwjxB9ANlKxerMNdfjqlTBWrVxgrjKi7VBwOY=
-github.com/openshift/library-go v0.0.0-20200327125526-163b2f0d6264/go.mod h1:CfydoH0B+RYs22uQZQ36A1mz5m5zhucpMGh8t5s71v4=
+github.com/openshift/library-go v0.0.0-20200402123743-4015ba624cae h1:QE9zls9gSMcB8LYWi2mFvVaiAcI4ECg6b45/OYHGe9E=
+github.com/openshift/library-go v0.0.0-20200402123743-4015ba624cae/go.mod h1:CfydoH0B+RYs22uQZQ36A1mz5m5zhucpMGh8t5s71v4=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/operator-framework/api v0.1.0/go.mod h1:/Jcpy9Ls8W1LK1hhEw30nCRBSBzbT8e/fE09TfRG0To=
 github.com/operator-framework/operator-registry v1.5.3/go.mod h1:agrQlkWOo1q8U1SAaLSS2WQ+Z9vswNT2M2HFib9iuLY=

pkg/cli/admin/release/extract.go

@@ -6,34 +6,45 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
 	digest "github.com/opencontainers/go-digest"
 	"github.com/spf13/cobra"
 	"k8s.io/klog"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	kcmdutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/templates"
 
 	"github.com/openshift/library-go/pkg/image/dockerv1client"
+	"github.com/openshift/library-go/pkg/manifest"
 	"github.com/openshift/oc/pkg/cli/image/extract"
 	"github.com/openshift/oc/pkg/cli/image/imagesource"
 	imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest"
 	"github.com/openshift/oc/pkg/cli/image/workqueue"
+	"github.com/pkg/errors"
 )
 
-func NewExtractOptions(streams genericclioptions.IOStreams) *ExtractOptions {
+// NewExtractOptions is also used internally as part of image mirroring. For image mirroring
+// internal use, extractManifests is set to true so image manifest files are searched for
+// signature information to be returned for use by mirroring.
+func NewExtractOptions(streams genericclioptions.IOStreams, extractManifests bool) *ExtractOptions {
 	return &ExtractOptions{
-		IOStreams: streams,
-		Directory: ".",
+		IOStreams:        streams,
+		Directory:        ".",
+		ExtractManifests: extractManifests,
 	}
 }
 
 func NewExtract(f kcmdutil.Factory, parentName string, streams genericclioptions.IOStreams) *cobra.Command {
-	o := NewExtractOptions(streams)
+	o := NewExtractOptions(streams, false)
 	cmd := &cobra.Command{
 		Use:   "extract",
 		Short: "Extract the contents of an update payload to disk",
@@ -110,6 +121,9 @@ type ExtractOptions struct {
 	File      string
 	FileDir   string
 
+	ExtractManifests bool
+	Manifests        []manifest.Manifest
+
 	ImageMetadataCallback func(m *extract.Mapping, dgst, contentDigest digest.Digest, config *dockerv1client.DockerImageConfig)
 }
 
@@ -196,23 +210,79 @@ func (o *ExtractOptions) Run() error {
 				To:   dir,
 			},
 		}
+		var manifestErrs []error
 		found := false
 		opts.TarEntryCallback = func(hdr *tar.Header, _ extract.LayerInfo, r io.Reader) (bool, error) {
-			if hdr.Name != o.File {
+			if !o.ExtractManifests {
+				if hdr.Name != o.File {
+					return true, nil
+				}
+				if _, err := io.Copy(o.Out, r); err != nil {
+					return false, err
+				}
+				found = true
+				return false, nil
+			} else {
+				switch hdr.Name {
+				case o.File:
+					if _, err := io.Copy(o.Out, r); err != nil {
+						return false, err
+					}
+					found = true
+				case "image-references":
+					return true, nil
+				case "release-metadata":
+					return true, nil
+				default:
+					if ext := path.Ext(hdr.Name); len(ext) > 0 && (ext == ".yaml" || ext == ".yml" || ext == ".json") {
+						klog.V(4).Infof("Found manifest %s", hdr.Name)
+						raw, err := ioutil.ReadAll(r)
+						if err != nil {
+							manifestErrs = append(manifestErrs, errors.Wrapf(err, "error reading file %s", hdr.Name))
+						}
+						ms, err := manifest.ParseManifests(bytes.NewReader(raw))
+						if err != nil {
+							manifestErrs = append(manifestErrs, errors.Wrapf(err, "error parsing %s", hdr.Name))
+						}
+						for i := range ms {
+							ms[i].OriginalFilename = filepath.Base(hdr.Name)
+							src := fmt.Sprintf("the config map %s/%s", ms[i].Obj.GetNamespace(), ms[i].Obj.GetName())
+							data, _, err := unstructured.NestedStringMap(ms[i].Obj.Object, "data")
+							if err != nil {
+								manifestErrs = append(manifestErrs, errors.Wrapf(err, "%s is not valid", src))
+							}
+							for k, v := range data {
+								switch {
+								case strings.HasPrefix(k, "verifier-public-key-"):
+									klog.Infof("Found in %s:\n%s %s", hdr.Name, k, v)
+									klog.V(2).Infof("Found in %s:\n%s %s", hdr.Name, k, v)
+								case strings.HasPrefix(k, "store-"):
+									klog.Infof("Found in %s:\n%s\n%s", hdr.Name, k, v)
+									klog.V(2).Infof("Found in %s:\n%s\n%s", hdr.Name, k, v)
+								default:
+								}
+							}
+						}
+						o.Manifests = append(o.Manifests, ms...)
+					}
+				}
 				return true, nil
 			}
-			if _, err := io.Copy(o.Out, r); err != nil {
-				return false, err
-			}
-			found = true
-			return false, nil
 		}
 		if err := opts.Run(); err != nil {
 			return err
 		}
 		if !found {
 			return fmt.Errorf("image did not contain %s", o.File)
 		}
+		// Warn user and print errors if any but do not return an error so current
+		// operation, e.g. mirroring, continues
+		if o.ExtractManifests && len(o.Manifests) == 0 {
+			fmt.Fprintf(o.ErrOut, "No manifests found\n")
+			if len(manifestErrs) > 0 {
+				fmt.Fprintf(o.ErrOut, "Errors: %s\n", errorList(manifestErrs))
+			}
+		}
 		return nil
 
 	default: