Skip to content

Commit

Permalink
WIP: implement enhancements/oc/mirroring-manifests
Browse files Browse the repository at this point in the history
  • Loading branch information
@jottofar
jottofar committed Mar 6, 2020
1 parent 8ce6126 commit 9b712d0
Show file tree
Hide file tree
Showing 35 changed files with 2,232 additions and 112 deletions.
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.1.0 // indirect
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
golang.org/x/net v0.0.0-20200202094626-16171245cfb2
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
gopkg.in/ldap.v2 v2.5.1
k8s.io/api v0.17.1
k8s.io/apimachinery v0.17.1
k8s.io/api v0.17.2
k8s.io/apimachinery v0.17.2
k8s.io/apiserver v0.17.1
k8s.io/cli-runtime v0.17.0
k8s.io/client-go v8.0.0+incompatible
Expand Down
10 changes: 10 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,8 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang-migrate/migrate/v4 v4.6.2 h1:LDDOHo/q1W5UDj6PbkxdCv7lv9yunyZHXvxuwDkGo3k=
github.com/golang-migrate/migrate/v4 v4.6.2/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
Expand Down Expand Up @@ -401,6 +403,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
Expand Down Expand Up @@ -611,6 +615,8 @@ github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52/go.mod h1
github.com/openshift/api v0.0.0-20200116145750-0e2ff1e215dd/go.mod h1:fT6U/JfG8uZzemTRwZA2kBDJP5nWz7v05UHnty/D+pk=
github.com/openshift/api v0.0.0-20200205133042-34f0ec8dab87 h1:L/fZlWB7DdYCd09r9LvBa44xRH42Dx80ybxfN1h5C8Y=
github.com/openshift/api v0.0.0-20200205133042-34f0ec8dab87/go.mod h1:fT6U/JfG8uZzemTRwZA2kBDJP5nWz7v05UHnty/D+pk=
github.com/openshift/api v0.0.0-20200210091934-a0e53e94816b h1:BERD6sZj7w9Tt0RBpuw87AC0+SppyxEUgUG/Of5rI+E=
github.com/openshift/api v0.0.0-20200210091934-a0e53e94816b/go.mod h1:fT6U/JfG8uZzemTRwZA2kBDJP5nWz7v05UHnty/D+pk=
github.com/openshift/build-machinery-go v0.0.0-20200205161356-ef115f5adc73/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160 h1:V4E6yt4XWiBEPKnJbs/E8pgUq9AjZqzQfsL3eeT84Qs=
github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
Expand All @@ -632,6 +638,8 @@ github.com/openshift/kubernetes-kubectl v0.0.0-20200211153013-50adac736181 h1:4W
github.com/openshift/kubernetes-kubectl v0.0.0-20200211153013-50adac736181/go.mod h1:jIPrUAW656Vzn9wZCCe0PC+oTcu56u2HgFD21Xbfk1s=
github.com/openshift/library-go v0.0.0-20200206134157-b4c763d94dcf h1:KXxMZGtbwefe89jh+XdZRcj2UMO71jdIZhKM3CQefMQ=
github.com/openshift/library-go v0.0.0-20200206134157-b4c763d94dcf/go.mod h1:mLRZGYfPe9Kaum4pyhn6hTO3FTPjOZKFB0RxQSIxiuQ=
github.com/openshift/library-go v0.0.0-20200303185131-81598fff9efa h1:TSdE7Mft5a4C3/kB2Bnkmi0Dr9yqE/3JQ+4wXmPZzvc=
github.com/openshift/library-go v0.0.0-20200303185131-81598fff9efa/go.mod h1:HM1nXsKTup/9rYnmXdYbbN+8Ux8Xp39odoqZ4qvSyBo=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/operator-framework/api v0.1.0/go.mod h1:/Jcpy9Ls8W1LK1hhEw30nCRBSBzbT8e/fE09TfRG0To=
github.com/operator-framework/operator-registry v1.5.3/go.mod h1:agrQlkWOo1q8U1SAaLSS2WQ+Z9vswNT2M2HFib9iuLY=
Expand Down Expand Up @@ -877,6 +885,8 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down
91 changes: 81 additions & 10 deletions pkg/cli/admin/release/extract.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,46 @@ package release

import (
"archive/tar"
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"time"

digest "github.com/opencontainers/go-digest"
"github.com/spf13/cobra"
"k8s.io/klog"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/cli-runtime/pkg/genericclioptions"
kcmdutil "k8s.io/kubectl/pkg/cmd/util"
"k8s.io/kubectl/pkg/util/templates"

"github.com/openshift/library-go/pkg/image/dockerv1client"
"github.com/openshift/library-go/pkg/manifest"
"github.com/openshift/oc/pkg/cli/image/extract"
"github.com/openshift/oc/pkg/cli/image/imagesource"
imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest"
"github.com/pkg/errors"
)

func NewExtractOptions(streams genericclioptions.IOStreams) *ExtractOptions {
// NewExtractOptions is also used internally as part of image mirroring. For image mirroring
// internal use, extractManifests is set to true so image manifest files are searched for
// signature information to be returned for use by mirroring.
func NewExtractOptions(streams genericclioptions.IOStreams, extractManifests bool) *ExtractOptions {
return &ExtractOptions{
IOStreams: streams,
Directory: ".",
IOStreams: streams,
Directory: ".",
ExtractManifests: extractManifests,
}
}

func NewExtract(f kcmdutil.Factory, parentName string, streams genericclioptions.IOStreams) *cobra.Command {
o := NewExtractOptions(streams)
o := NewExtractOptions(streams, false)
cmd := &cobra.Command{
Use: "extract",
Short: "Extract the contents of an update payload to disk",
Expand Down Expand Up @@ -102,6 +114,9 @@ type ExtractOptions struct {
File string
FileDir string

ExtractManifests bool
Manifests []manifest.Manifest

ImageMetadataCallback func(m *extract.Mapping, dgst, contentDigest digest.Digest, config *dockerv1client.DockerImageConfig)
}

Expand Down Expand Up @@ -184,23 +199,79 @@ func (o *ExtractOptions) Run() error {
To: dir,
},
}
var manifestErrs []error
found := false
opts.TarEntryCallback = func(hdr *tar.Header, _ extract.LayerInfo, r io.Reader) (bool, error) {
if hdr.Name != o.File {
if !o.ExtractManifests {
if hdr.Name != o.File {
return true, nil
}
if _, err := io.Copy(o.Out, r); err != nil {
return false, err
}
found = true
return false, nil
} else {
switch hdr.Name {
case o.File:
if _, err := io.Copy(o.Out, r); err != nil {
return false, err
}
found = true
case "image-references":
return true, nil
case "release-metadata":
return true, nil
default:
if ext := path.Ext(hdr.Name); len(ext) > 0 && (ext == ".yaml" || ext == ".yml" || ext == ".json") {
klog.V(4).Infof("Found manifest %s", hdr.Name)
raw, err := ioutil.ReadAll(r)
if err != nil {
manifestErrs = append(manifestErrs, errors.Wrapf(err, "error reading file %s", hdr.Name))
}
ms, err := manifest.ParseManifests(bytes.NewReader(raw))
if err != nil {
manifestErrs = append(manifestErrs, errors.Wrapf(err, "error parsing %s", hdr.Name))
}
for i := range ms {
ms[i].OriginalFilename = filepath.Base(hdr.Name)
src := fmt.Sprintf("the config map %s/%s", ms[i].Obj.GetNamespace(), ms[i].Obj.GetName())
data, _, err := unstructured.NestedStringMap(ms[i].Obj.Object, "data")
if err != nil {
manifestErrs = append(manifestErrs, errors.Wrapf(err, "%s is not valid", src))
}
for k, v := range data {
switch {
case strings.HasPrefix(k, "verifier-public-key-"):
klog.Infof("Found in %s:\n%s %s", hdr.Name, k, v)
klog.V(2).Infof("Found in %s:\n%s %s", hdr.Name, k, v)
case strings.HasPrefix(k, "store-"):
klog.Infof("Found in %s:\n%s\n%s", hdr.Name, k, v)
klog.V(2).Infof("Found in %s:\n%s\n%s", hdr.Name, k, v)
default:
}
}
}
o.Manifests = append(o.Manifests, ms...)
}
}
return true, nil
}
if _, err := io.Copy(o.Out, r); err != nil {
return false, err
}
found = true
return false, nil
}
if err := opts.Run(); err != nil {
return err
}
if !found {
return fmt.Errorf("image did not contain %s", o.File)
}
// Warn user and print errors if any but do not return an error so current
// operation, e.g. mirroring, continues
if o.ExtractManifests && len(o.Manifests) == 0 {
fmt.Fprintf(o.ErrOut, "No manifests found\n")
if len(manifestErrs) > 0 {
fmt.Fprintf(o.ErrOut, "Errors: %s\n", errorList(manifestErrs))
}
}
return nil

default:
Expand Down
Loading

0 comments on commit 9b712d0

Please sign in to comment.