feat: support multiple registry mirrors with fallback

Fixes GoogleContainerTools#1473

The initial implementation of the registry mirror only allowed a single mirror, and if pulling from the mirror failed, the build would fail.

This change introduces:
- multiple registry mirrors instead of a single one
- fallback if an image can't be pulled from a registry

This is the same behavior as the docker daemon and will allow using a registry mirror such as `mirror.gcr.io` which is incomplete and doesn't have all the content that the default registry on docker.io has.

Note that there are no changes in the CLI flags, the `--registry-mirror` flag is still valid. But now it can be used multiple times to set up more than one registry mirror.

README.md

@@ -652,9 +652,9 @@ Expected format is `my.registry.url=/path/to/the/certificate.cert`
 
 #### --registry-mirror
 
-Set this flag if you want to use a registry mirror instead of default `index.docker.io`.
-
+Set this flag if you want to use a registry mirror instead of the default `index.docker.io`. You can use this flag more than once, if you want to set multiple mirrors. If an image is not found on the first mirror, Kaniko will try the next mirror(s), and at the end fallback on the default registry.
 
+Expected format is `mirror.gcr.io` for example.
 
 #### --reproducible
 

cmd/executor/cmd/root.go

@@ -172,7 +172,7 @@ func addKanikoOptionsFlags() {
  RootCmd.PersistentFlags().VarP(&opts.SkipTLSVerifyRegistries, "skip-tls-verify-registry", "", "Insecure registry ignoring TLS verify to push and pull. Set it repeatedly for multiple registries.")
  opts.RegistriesCertificates = make(map[string]string)
  RootCmd.PersistentFlags().VarP(&opts.RegistriesCertificates, "registry-certificate", "", "Use the provided certificate for TLS communication with the given registry. Expected format is 'my.registry.url=/path/to/the/server/certificate'.")
- RootCmd.PersistentFlags().StringVarP(&opts.RegistryMirror, "registry-mirror", "", "", "Registry mirror to use has pull-through cache instead of docker.io.")
+ RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.")
  RootCmd.PersistentFlags().BoolVarP(&opts.IgnoreVarRun, "whitelist-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image. (Default true).")
  RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.")
  RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")

integration/integration_test.go

@@ -304,7 +304,7 @@ func TestGitBuildcontextSubPath(t *testing.T) {
  checkContainerDiffOutput(t, diff, expected)
 }
 
-func TestBuildViaRegistryMirror(t *testing.T) {
+func TestBuildViaRegistryMirrors(t *testing.T) {
  repo := getGitRepo()
  dockerfile := fmt.Sprintf("%s/%s/Dockerfile_registry_mirror", integrationPath, dockerfilesPath)
 
@@ -327,6 +327,7 @@ func TestBuildViaRegistryMirror(t *testing.T) {
  dockerRunFlags = append(dockerRunFlags, ExecutorImage,
  "-f", dockerfile,
  "-d", kanikoImage,
+ "--registry-mirror", "doesnotexist.example.com",
  "--registry-mirror", "us-mirror.gcr.io",
  "-c", fmt.Sprintf("git://%s", repo))
 

pkg/config/options.go

@@ -43,7 +43,7 @@ type KanikoOptions struct {
  DigestFile string
  ImageNameDigestFile string
  OCILayoutPath string
- RegistryMirror  string
+ RegistryMirrors multiArg
  Destinations multiArg
  BuildArgs multiArg
  InsecureRegistries multiArg

pkg/image/image_util.go

@@ -106,51 +106,45 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
  return nil, err
  }
 
- registryName := ref.Context().RegistryStr()
- var newReg name.Registry
- toSet := false
-
- if opts.RegistryMirror != "" && registryName == name.DefaultRegistry {
- registryName = opts.RegistryMirror
-
- newReg, err = name.NewRegistry(opts.RegistryMirror, name.StrictValidation)
+ if ref.Context().RegistryStr() == name.DefaultRegistry {
+ ref, err := normalizeReference(ref, image)
  if err != nil {
  return nil, err
  }
 
- ref, err = normalizeReference(ref, image)
- if err != nil {
- return nil, err
- }
+ for _, registryMirror := range opts.RegistryMirrors {
+ var newReg name.Registry
+ if opts.InsecurePull || opts.InsecureRegistries.Contains(registryMirror) {
+ newReg, err = name.NewRegistry(registryMirror, name.WeakValidation, name.Insecure)
+ } else {
+ newReg, err = name.NewRegistry(registryMirror, name.StrictValidation)
+ }
+ if err != nil {
+ return nil, err
+ }
+ ref := setNewRegistry(ref, newReg)
 
- toSet = true
+ logrus.Infof("Retrieving image %s from registry mirror %s", ref, registryMirror)
+ remoteImage, err := remote.Image(ref, remoteOptions(registryMirror, opts)...)
+ if err != nil {
+ logrus.Warnf("Failed to retrieve image %s from registry mirror %s: %s. Will try with the next mirror, or fallback to the default registry.", ref, registryMirror, err)
+ continue
+ }
+ return remoteImage, nil
+ }
  }
 
+ registryName := ref.Context().RegistryStr()
  if opts.InsecurePull || opts.InsecureRegistries.Contains(registryName) {
- newReg, err = name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
+ newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
  if err != nil {
  return nil, err
  }
-
- toSet = true
- }
-
- if toSet {
- if tag, ok := ref.(name.Tag); ok {
- tag.Repository.Registry = newReg
- ref = tag
- }
-
- if digest, ok := ref.(name.Digest); ok {
- digest.Repository.Registry = newReg
- ref = digest
- }
+ ref = setNewRegistry(ref, newReg)
  }
 
- logrus.Infof("Retrieving image %s", ref)
-
- rOpts := remoteOptions(registryName, opts)
- return remote.Image(ref, rOpts...)
+ logrus.Infof("Retrieving image %s from registry %s", ref, registryName)
+ return remote.Image(ref, remoteOptions(registryName, opts)...)
 }
 
 // normalizeReference adds the library/ prefix to images without it.
@@ -165,6 +159,19 @@ func normalizeReference(ref name.Reference, image string) (name.Reference, error
  return ref, nil
 }
 
+func setNewRegistry(ref name.Reference, newReg name.Registry) name.Reference {
+ switch r := ref.(type) {
+ case name.Tag:
+ r.Repository.Registry = newReg
+ return r
+ case name.Digest:
+ r.Repository.Registry = newReg
+ return r
+ default:
+ return ref
+ }
+}
+
 func remoteOptions(registryName string, opts *config.KanikoOptions) []remote.Option {
  tr := util.MakeTransport(opts, registryName)
 

pkg/image/image_util_test.go

@@ -104,7 +104,7 @@ func Test_ScratchImageFromMirror(t *testing.T) {
  actual, err := RetrieveSourceImage(config.KanikoStage{
  Stage: stages[1],
  }, &config.KanikoOptions{
- RegistryMirror: "mirror.gcr.io",
+ RegistryMirrors: []string{"mirror.gcr.io"},
  })
  expected := empty.Image
  testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual)