feat(warmer): Warmer now supports all registry-related flags

which means we can now:
- set up one or more mirrors
- set up registries certificates
- skip TLS verify
- use plain HTTP
using the same set of flags that are defined for the executor

cmd/warmer/cmd/root.go

@@ -75,6 +75,14 @@ func addKanikoOptionsFlags() {
  RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.")
  RootCmd.PersistentFlags().BoolVarP(&opts.Force, "force", "f", false, "Force cache overwriting.")
  RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.")
+ RootCmd.PersistentFlags().BoolVarP(&opts.InsecurePull, "insecure-pull", "", false, "Pull from insecure registry using plain HTTP")
+ RootCmd.PersistentFlags().BoolVarP(&opts.SkipTLSVerifyPull, "skip-tls-verify-pull", "", false, "Pull from insecure registry ignoring TLS verify")
+ RootCmd.PersistentFlags().VarP(&opts.InsecureRegistries, "insecure-registry", "", "Insecure registry using plain HTTP to pull. Set it repeatedly for multiple registries.")
+ RootCmd.PersistentFlags().VarP(&opts.SkipTLSVerifyRegistries, "skip-tls-verify-registry", "", "Insecure registry ignoring TLS verify to pull. Set it repeatedly for multiple registries.")
+ opts.RegistriesCertificates = make(map[string]string)
+ RootCmd.PersistentFlags().VarP(&opts.RegistriesCertificates, "registry-certificate", "", "Use the provided certificate for TLS communication with the given registry. Expected format is 'my.registry.url=/path/to/the/server/certificate'.")
+ RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.")
+ RootCmd.PersistentFlags().StringVarP(&opts.CustomPlatform, "customPlatform", "", "", "Specify the build platform if different from the current host")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

pkg/cache/cache.go

@@ -66,7 +66,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
  cacheRef.Repository.Registry = newReg
  }
 
- tr := util.MakeTransport(rc.Opts, registryName)
+ tr := util.MakeTransport(rc.Opts.RegistryOptions, registryName)
 
  img, err := remote.Image(cacheRef, remote.WithTransport(tr), remote.WithAuthFromKeychain(creds.GetKeychain()))
  if err != nil {

pkg/cache/doc_test.go

@@ -21,14 +21,14 @@ import (
  "log"
 
  "github.com/GoogleContainerTools/kaniko/pkg/config"
- "github.com/google/go-containerregistry/pkg/v1/remote"
+ "github.com/GoogleContainerTools/kaniko/pkg/image/remote"
 )
 
 func ExampleWarmer_Warm() {
  tarBuf := new(bytes.Buffer)
  manifestBuf := new(bytes.Buffer)
  w := &Warmer{
- Remote: remote.Image,
+ Remote: remote.RetrieveRemoteImage,
  Local: LocalSource,
  TarWriter: tarBuf,
  ManifestWriter: manifestBuf,

pkg/cache/warm.go

@@ -20,16 +20,13 @@ import (
  "bytes"
  "io"
  "io/ioutil"
- "net/http"
  "os"
  "path"
- "runtime"
 
  "github.com/GoogleContainerTools/kaniko/pkg/config"
- "github.com/GoogleContainerTools/kaniko/pkg/creds"
+ "github.com/GoogleContainerTools/kaniko/pkg/image/remote"
  "github.com/google/go-containerregistry/pkg/name"
  v1 "github.com/google/go-containerregistry/pkg/v1"
- "github.com/google/go-containerregistry/pkg/v1/remote"
  "github.com/google/go-containerregistry/pkg/v1/tarball"
  "github.com/pkg/errors"
  "github.com/sirupsen/logrus"
@@ -42,18 +39,18 @@ func WarmCache(opts *config.WarmerOptions) error {
  logrus.Debugf("%s\n", cacheDir)
  logrus.Debugf("%s\n", images)
 
- for _, image := range images {
+ for _, img := range images {
  tarBuf := new(bytes.Buffer)
  manifestBuf := new(bytes.Buffer)
 
  cw := &Warmer{
- Remote: remote.Image,
+ Remote: remote.RetrieveRemoteImage,
  Local: LocalSource,
  TarWriter: tarBuf,
  ManifestWriter: manifestBuf,
  }
 
- digest, err := cw.Warm(image, opts)
+ digest, err := cw.Warm(img, opts)
  if err != nil {
  if !IsAlreadyCached(err) {
  return err
@@ -68,7 +65,7 @@ func WarmCache(opts *config.WarmerOptions) error {
  return err
  }
 
- logrus.Debugf("Wrote %s to cache", image)
+ logrus.Debugf("Wrote %s to cache", img)
  }
  return nil
 }
@@ -93,9 +90,9 @@ func writeBufsToFile(cachePath string, tarBuf, manifestBuf *bytes.Buffer) error
 }
 
 // FetchRemoteImage retrieves a Docker image manifest from a remote source.
-// github.com/google/go-containerregistry/pkg/v1/remote.Image can be used as
+// github.com/GoogleContainerTools/kaniko/image/remote.RetrieveRemoteImage can be used as
 // this type.
-type FetchRemoteImage func(name.Reference, ...remote.Option) (v1.Image, error)
+type FetchRemoteImage func(image string, opts config.RegistryOptions, customPlatform string) (v1.Image, error)
 
 // FetchLocalSource retrieves a Docker image manifest from a local source.
 // github.com/GoogleContainerTools/kaniko/cache.LocalSource can be used as
@@ -118,11 +115,7 @@ func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error)
  return v1.Hash{}, errors.Wrapf(err, "Failed to verify image name: %s", image)
  }
 
- transport := http.DefaultTransport.(*http.Transport)
- platform := currentPlatform()
-
- rOpts := []remote.Option{remote.WithTransport(transport), remote.WithAuthFromKeychain(creds.GetKeychain()), remote.WithPlatform(platform)}
- img, err := w.Remote(cacheRef, rOpts...)
+ img, err := w.Remote(image, opts.RegistryOptions, opts.CustomPlatform)
  if err != nil || img == nil {
  return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve image: %s", image)
  }
@@ -155,11 +148,3 @@ func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error)
 
  return digest, nil
 }
-
-// CurrentPlatform returns the v1.Platform on which the code runs.
-func currentPlatform() v1.Platform {
- return v1.Platform{
- OS: runtime.GOOS,
- Architecture: runtime.GOARCH,
- }
-}

pkg/cache/warm_test.go

@@ -22,9 +22,7 @@ import (
 
  "github.com/GoogleContainerTools/kaniko/pkg/config"
  "github.com/GoogleContainerTools/kaniko/pkg/fakes"
- "github.com/google/go-containerregistry/pkg/name"
  v1 "github.com/google/go-containerregistry/pkg/v1"
- "github.com/google/go-containerregistry/pkg/v1/remote"
 )
 
 const (
@@ -36,7 +34,7 @@ func Test_Warmer_Warm_not_in_cache(t *testing.T) {
  manifestBuf := new(bytes.Buffer)
 
  cw := &Warmer{
- Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+ Remote: func(_ string, _ config.RegistryOptions, _ string) (v1.Image, error) {
  return fakes.FakeImage{}, nil
  },
  Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {
@@ -64,7 +62,7 @@ func Test_Warmer_Warm_in_cache_not_expired(t *testing.T) {
  manifestBuf := new(bytes.Buffer)
 
  cw := &Warmer{
- Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+ Remote: func(_ string, _ config.RegistryOptions, _ string) (v1.Image, error) {
  return fakes.FakeImage{}, nil
  },
  Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {
@@ -92,7 +90,7 @@ func Test_Warmer_Warm_in_cache_expired(t *testing.T) {
  manifestBuf := new(bytes.Buffer)
 
  cw := &Warmer{
- Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+ Remote: func(_ string, _ config.RegistryOptions, _ string) (v1.Image, error) {
  return fakes.FakeImage{}, nil
  },
  Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {

pkg/config/options.go

@@ -30,40 +30,45 @@ type CacheOptions struct {
  CacheTTL time.Duration
 }
 
-// KanikoOptions are options that are set by command line arguments
-type KanikoOptions struct {
- CacheOptions
- DockerfilePath string
- SrcContext string
- SnapshotMode string
- CustomPlatform string
- Bucket string
- TarPath string
- Target string
- CacheRepo string
- DigestFile string
- ImageNameDigestFile string
- OCILayoutPath string
+// RegistryOptions are all the options related to the registries, set by command line arguments.
+type RegistryOptions struct {
  RegistryMirrors multiArg
- Destinations multiArg
- BuildArgs multiArg
  InsecureRegistries multiArg
- Labels multiArg
  SkipTLSVerifyRegistries multiArg
  RegistriesCertificates keyValueArg
  Insecure bool
  SkipTLSVerify bool
  InsecurePull bool
  SkipTLSVerifyPull bool
- SingleSnapshot bool
- Reproducible bool
- NoPush bool
- Cache bool
- Cleanup bool
- IgnoreVarRun bool
- SkipUnusedStages bool
- RunV2 bool
- Git KanikoGitOptions
+}
+
+// KanikoOptions are options that are set by command line arguments
+type KanikoOptions struct {
+ CacheOptions
+ RegistryOptions
+ DockerfilePath string
+ SrcContext string
+ SnapshotMode string
+ CustomPlatform string
+ Bucket string
+ TarPath string
+ Target string
+ CacheRepo string
+ DigestFile string
+ ImageNameDigestFile string
+ OCILayoutPath string
+ Destinations multiArg
+ BuildArgs multiArg
+ Labels multiArg
+ SingleSnapshot bool
+ Reproducible bool
+ NoPush bool
+ Cache bool
+ Cleanup bool
+ IgnoreVarRun bool
+ SkipUnusedStages bool
+ RunV2 bool
+ Git KanikoGitOptions
 }
 
 type KanikoGitOptions struct {
@@ -109,6 +114,8 @@ func (k *KanikoGitOptions) Set(s string) error {
 // WarmerOptions are options that are set by command line arguments to the cache warmer.
 type WarmerOptions struct {
  CacheOptions
- Images multiArg
- Force bool
+ RegistryOptions
+ CustomPlatform string
+ Images multiArg
+ Force bool
 }

pkg/executor/build.go

@@ -25,29 +25,27 @@ import (
  "strings"
  "time"
 
- "github.com/google/go-containerregistry/pkg/v1/partial"
-
- "github.com/moby/buildkit/frontend/dockerfile/instructions"
-
- "golang.org/x/sync/errgroup"
-
  "github.com/google/go-containerregistry/pkg/name"
  v1 "github.com/google/go-containerregistry/pkg/v1"
  "github.com/google/go-containerregistry/pkg/v1/empty"
  "github.com/google/go-containerregistry/pkg/v1/mutate"
  "github.com/google/go-containerregistry/pkg/v1/tarball"
+ "github.com/moby/buildkit/frontend/dockerfile/instructions"
  "github.com/pkg/errors"
  "github.com/sirupsen/logrus"
+ "golang.org/x/sync/errgroup"
 
  "github.com/GoogleContainerTools/kaniko/pkg/cache"
  "github.com/GoogleContainerTools/kaniko/pkg/commands"
  "github.com/GoogleContainerTools/kaniko/pkg/config"
  "github.com/GoogleContainerTools/kaniko/pkg/constants"
  "github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
  image_util "github.com/GoogleContainerTools/kaniko/pkg/image"
+ "github.com/GoogleContainerTools/kaniko/pkg/image/remote"
  "github.com/GoogleContainerTools/kaniko/pkg/snapshot"
  "github.com/GoogleContainerTools/kaniko/pkg/timing"
  "github.com/GoogleContainerTools/kaniko/pkg/util"
+ "github.com/google/go-containerregistry/pkg/v1/partial"
 )
 
 // This is the size of an empty tar in Go
@@ -745,7 +743,7 @@ func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) e
 
  // This must be an image name, fetch it.
  logrus.Debugf("Found extra base image stage %s", c.From)
- sourceImage, err := image_util.RetrieveRemoteImage(c.From, opts)
+ sourceImage, err := remote.RetrieveRemoteImage(c.From, opts.RegistryOptions, opts.CustomPlatform)
  if err != nil {
  return err
  }

pkg/executor/push.go

@@ -147,7 +147,7 @@ func CheckPushPermissions(opts *config.KanikoOptions) error {
  }
  destRef.Repository.Registry = newReg
  }
- tr := newRetry(util.MakeTransport(opts, registryName))
+ tr := newRetry(util.MakeTransport(opts.RegistryOptions, registryName))
  if err := checkRemotePushPermission(destRef, creds.GetKeychain(), tr); err != nil {
  return errors.Wrapf(err, "checking push permission for %q", destRef)
  }
@@ -244,7 +244,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error {
  return errors.Wrap(err, "resolving pushAuth")
  }
 
- tr := newRetry(util.MakeTransport(opts, registryName))
+ tr := newRetry(util.MakeTransport(opts.RegistryOptions, registryName))
  rt := &withUserAgent{t: tr}
 
  if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil {