Implement the OverwriteRegistry functionality (#1145)

* Add the initial RegistryConfiguration API

* Update example config manifest

* Make templates use OverwriteImage functionality

* Use OverwriteRegistry for kubeadm

* Add Registry function to addons templating

* Make addons use the Registry template function

* Update addons manifest tests

addons/backups-restic/backups-restic.yaml

@@ -48,7 +48,7 @@ spec:
               path: /etc/kubernetes/pki
           initContainers:
           - name: snapshoter
-            image: gcr.io/etcd-development/etcd:v3.4.3
+            image: {{ Registry "gcr.io" }}/etcd-development/etcd:v3.4.3
             imagePullPolicy: IfNotPresent
             command:
             - /bin/sh
@@ -89,7 +89,7 @@ spec:
               readOnly: true
           containers:
           - name: uploader
-            image: docker.io/restic/restic:0.9.6
+            image: {{ Registry "docker.io" }}/restic/restic:0.9.6
             imagePullPolicy: IfNotPresent
             command:
             - /bin/sh

addons/calico-vxlan/calico-vxlan.yaml

@@ -3458,7 +3458,7 @@ spec:
         # It can be deleted if this is a fresh installation, or if you have already
         # upgraded to use calico-ipam.
         - name: upgrade-ipam
-          image: calico/cni:v3.15.1
+          image: {{ Registry "docker.io" }}/calico/cni:v3.15.1
           command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
           env:
             - name: KUBERNETES_NODE_NAME
@@ -3480,7 +3480,7 @@ spec:
         # This container installs the CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
-          image: calico/cni:v3.15.1
+          image: {{ Registry "docker.io" }}/calico/cni:v3.15.1
           command: ["/install-cni.sh"]
           env:
             # Name of the CNI config file to create.
@@ -3516,7 +3516,7 @@ spec:
         # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
         # to communicate with Felix over the Policy Sync API.
         - name: flexvol-driver
-          image: calico/pod2daemon-flexvol:v3.15.1
+          image: {{ Registry "docker.io" }}/calico/pod2daemon-flexvol:v3.15.1
           volumeMounts:
           - name: flexvol-driver-host
             mountPath: /host/driver
@@ -3527,7 +3527,7 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: calico/node:v3.15.1
+          image: {{ Registry "docker.io" }}/calico/node:v3.15.1
           env:
             # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
@@ -3712,7 +3712,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: calico-kube-controllers
-          image: calico/kube-controllers:v3.15.1
+          image: {{ Registry "docker.io" }}/calico/kube-controllers:v3.15.1
           env:
             # Choose which controllers to run.
             - name: ENABLED_CONTROLLERS

addons/cluster-autoscaler/cluster-autoscaler.yaml

@@ -146,7 +146,7 @@ spec:
         app: cluster-autoscaler
     spec:
       containers:
-      - image: k8s.gcr.io/autoscaling/cluster-autoscaler:${AUTOSCALER_VERSION}
+      - image: {{ Registry "k8s.gcr.io" }}/autoscaling/cluster-autoscaler:${AUTOSCALER_VERSION}
         name: cluster-autoscaler
         command:
         - /cluster-autoscaler

pkg/addons/manifest.go

@@ -47,7 +47,12 @@ func getManifests(s *state.State, templateData TemplateData) error {
 		addonsPath = filepath.Join(manifestAbsPath, addonsPath)
 	}
 
-	manifests, err := loadAddonsManifests(addonsPath, s.Logger, s.Verbose, templateData)
+	overwriteRegistry := ""
+	if s.Cluster.RegistryConfiguration != nil && s.Cluster.RegistryConfiguration.OverwriteRegistry != "" {
+		overwriteRegistry = s.Cluster.RegistryConfiguration.OverwriteRegistry
+	}
+
+	manifests, err := loadAddonsManifests(addonsPath, s.Logger, s.Verbose, templateData, overwriteRegistry)
 	if err != nil {
 		return err
 	}
@@ -64,7 +69,7 @@ func getManifests(s *state.State, templateData TemplateData) error {
 }
 
 // loadAddonsManifests loads all YAML files from a given directory and runs the templating logic
-func loadAddonsManifests(addonsPath string, logger logrus.FieldLogger, verbose bool, templateData TemplateData) ([]runtime.RawExtension, error) {
+func loadAddonsManifests(addonsPath string, logger logrus.FieldLogger, verbose bool, templateData TemplateData, overwriteRegistry string) ([]runtime.RawExtension, error) {
 	manifests := []runtime.RawExtension{}
 
 	files, err := ioutil.ReadDir(addonsPath)
@@ -97,7 +102,7 @@ func loadAddonsManifests(addonsPath string, logger logrus.FieldLogger, verbose b
 			return nil, errors.Wrapf(err, "failed to load addon %s", file.Name())
 		}
 
-		tpl, err := template.New("addons-base").Funcs(sprig.TxtFuncMap()).Parse(string(manifestBytes))
+		tpl, err := template.New("addons-base").Funcs(txtFuncMap(overwriteRegistry)).Parse(string(manifestBytes))
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to template addons manifest %s", file.Name())
 		}
@@ -188,3 +193,15 @@ func combineManifests(manifests []*bytes.Buffer) *bytes.Buffer {
 
 	return bytes.NewBufferString(strings.Join(parts, "\n---\n") + "\n")
 }
+
+func txtFuncMap(overwriteRegistry string) template.FuncMap {
+	funcs := sprig.TxtFuncMap()
+	funcs["Registry"] = func(registry string) string {
+		if overwriteRegistry != "" {
+			return overwriteRegistry
+		}
+		return registry
+	}
+
+	return funcs
+}

pkg/addons/manifest_test.go

@@ -87,6 +87,50 @@ metadata:
 `
 )
 
+const (
+	testManifest1WithImage = `apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: test
+    cluster: kubeone-test
+    kubeone.io/addon: ""
+  name: test1
+  namespace: kube-system
+spec:
+  containers:
+  - image: {{ Registry "k8s.gcr.io" }}/kube-apiserver:v1.19.3
+`
+
+	testManifest1WithImageParsed = `apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: test
+    cluster: kubeone-test
+    kubeone.io/addon: ""
+  name: test1
+  namespace: kube-system
+spec:
+  containers:
+  - image: k8s.gcr.io/kube-apiserver:v1.19.3
+`
+
+	testManifest1WithCustomImageParsed = `apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: test
+    cluster: kubeone-test
+    kubeone.io/addon: ""
+  name: test1
+  namespace: kube-system
+spec:
+  containers:
+  - image: 127.0.0.1:5000/kube-apiserver:v1.19.3
+`
+)
+
 var (
 	// testManifest1 & testManifest3 have a linebreak at the end, testManifest2 not
 	combinedTestManifest = fmt.Sprintf("%s---\n%s\n---\n%s", testManifests[0], testManifests[1], testManifests[2])
@@ -108,7 +152,7 @@ func TestEnsureAddonsLabelsOnResources(t *testing.T) {
 			Name: "kubeone-test",
 		},
 	}
-	manifests, err := loadAddonsManifests(addonsDir, nil, false, templateData)
+	manifests, err := loadAddonsManifests(addonsDir, nil, false, templateData, "")
 	if err != nil {
 		t.Fatalf("unable to load manifests: %v", err)
 	}
@@ -139,3 +183,72 @@ func TestCombineManifests(t *testing.T) {
 		t.Fatalf("invalid combined manifest returned. expected \n%s, got \n%s", combinedTestManifest, manifest.String())
 	}
 }
+
+func TestImageRegistryParsing(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		registryConfigurtion *kubeoneapi.RegistryConfiguration
+		inputManifest        string
+		expectedManifest     string
+	}{
+		{
+			name:                 "default registry configuration",
+			registryConfigurtion: nil,
+			inputManifest:        testManifest1WithImage,
+			expectedManifest:     testManifest1WithImageParsed,
+		},
+		{
+			name: "custom registry",
+			registryConfigurtion: &kubeoneapi.RegistryConfiguration{
+				OverwriteRegistry: "127.0.0.1:5000",
+			},
+			inputManifest:    testManifest1WithImage,
+			expectedManifest: testManifest1WithCustomImageParsed,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			addonsDir, err := ioutil.TempDir("/tmp", "kubeone")
+			if err != nil {
+				t.Fatalf("unable to create temporary addons directory: %v", err)
+			}
+			defer os.RemoveAll(addonsDir)
+
+			if writeErr := ioutil.WriteFile(path.Join(addonsDir, "testManifest.yaml"), []byte(tc.inputManifest), 0600); writeErr != nil {
+				t.Fatalf("unable to create temporary addon manifest: %v", err)
+			}
+
+			templateData := TemplateData{
+				Config: &kubeoneapi.KubeOneCluster{
+					Name:                  "kubeone-test",
+					RegistryConfiguration: tc.registryConfigurtion,
+				},
+			}
+
+			overwriteRegistry := ""
+			if tc.registryConfigurtion != nil && tc.registryConfigurtion.OverwriteRegistry != "" {
+				overwriteRegistry = tc.registryConfigurtion.OverwriteRegistry
+			}
+
+			manifests, err := loadAddonsManifests(addonsDir, nil, false, templateData, overwriteRegistry)
+			if err != nil {
+				t.Fatalf("unable to load manifests: %v", err)
+			}
+			if len(manifests) != 1 {
+				t.Fatalf("expected to load 1 manifest, got %d", len(manifests))
+			}
+
+			b, err := ensureAddonsLabelsOnResources(manifests)
+			if err != nil {
+				t.Fatalf("unable to ensure labels: %v", err)
+			}
+			manifest := b[0].String()
+
+			if manifest != tc.expectedManifest {
+				t.Fatalf("invalid manifest returned. expected \n%s, got \n%s", tc.expectedManifest, manifest)
+			}
+		})
+	}
+}

pkg/apis/kubeone/helpers.go

@@ -104,3 +104,12 @@ func (p CloudProviderSpec) CloudProviderInTree() bool {
 
 	return false
 }
+
+// ImageRegistry returns the image registry to use or the passed in
+// default if no override is specified
+func (r *RegistryConfiguration) ImageRegistry(defaultRegistry string) string {
+	if r != nil && r.OverwriteRegistry != "" {
+		return r.OverwriteRegistry
+	}
+	return defaultRegistry
+}

pkg/apis/kubeone/types.go

@@ -57,6 +57,8 @@ type KubeOneCluster struct {
 	Addons *Addons `json:"addons,omitempty"`
 	// SystemPackages configure kubeone behaviour regarding OS packages.
 	SystemPackages *SystemPackages `json:"systemPackages,omitempty"`
+	// RegistryConfiguration configures how Docker images are pulled from an image registry
+	RegistryConfiguration *RegistryConfiguration `json:"registryConfiguration,omitempty"`
 }
 
 // ContainerRuntimeConfig
@@ -364,6 +366,19 @@ type SystemPackages struct {
 	ConfigureRepositories bool `json:"configureRepositories,omitempty"`
 }
 
+// RegistryConfiguration controls how images used for components deployed by
+// KubeOne and kubeadm are pulled from an image registry
+type RegistryConfiguration struct {
+	// OverwriteRegistry specifies a custom Docker registry which will be used
+	// for all images required for KubeOne and kubeadm. This also applies to
+	// addons deployed by KubeOne.
+	// This field doesn't modify the user/organization part of the image. For example,
+	// if OverwriteRegistry is set to 127.0.0.1:5000/example, image called
+	// calico/cni would translate to 127.0.0.1:5000/example/calico/cni.
+	// Default: ""
+	OverwriteRegistry string `json:"overwriteRegistry,omitempty"`
+}
+
 // PodNodeSelector feature flag
 type PodNodeSelector struct {
 	// Enable

pkg/apis/kubeone/v1beta1/types.go

@@ -57,6 +57,8 @@ type KubeOneCluster struct {
 	Addons *Addons `json:"addons,omitempty"`
 	// SystemPackages configure kubeone behaviour regarding OS packages.
 	SystemPackages *SystemPackages `json:"systemPackages,omitempty"`
+	// RegistryConfiguration configures how Docker images are pulled from an image registry
+	RegistryConfiguration *RegistryConfiguration `json:"registryConfiguration,omitempty"`
 }
 
 // ContainerRuntimeConfig
@@ -364,6 +366,19 @@ type SystemPackages struct {
 	ConfigureRepositories bool `json:"configureRepositories,omitempty"`
 }
 
+// RegistryConfiguration controls how images used for components deployed by
+// KubeOne and kubeadm are pulled from an image registry
+type RegistryConfiguration struct {
+	// OverwriteRegistry specifies a custom Docker registry which will be used
+	// for all images required for KubeOne and kubeadm. This also applies to
+	// addons deployed by KubeOne.
+	// This field doesn't modify the user/organization part of the image. For example,
+	// if OverwriteRegistry is set to 127.0.0.1:5000/example, image called
+	// calico/cni would translate to 127.0.0.1:5000/example/calico/cni.
+	// Default: ""
+	OverwriteRegistry string `json:"overwriteRegistry,omitempty"`
+}
+
 // PodNodeSelector feature flag
 type PodNodeSelector struct {
 	// Enable