[installer]: add image pull secrets and custom image registry

installer/pkg/common/common.go

@@ -264,10 +264,10 @@ func MessageBusWaiterContainer(ctx *RenderContext) *corev1.Container {
 	}
 }
 
-func KubeRBACProxyContainer() *corev1.Container {
+func KubeRBACProxyContainer(ctx *RenderContext) *corev1.Container {
 	return &corev1.Container{
 		Name:  "kube-rbac-proxy",
-		Image: "quay.io/brancz/kube-rbac-proxy:v0.11.0",
+		Image: ImageName(ThirdPartyContainerRepo(ctx.Config.Repository, KubeRBACProxyRepo), KubeRBACProxyImage, KubeRBACProxyTag),
 		Args: []string{
 			"--v=5",
 			"--logtostderr",
@@ -509,3 +509,17 @@ func RandomString(length int) (string, error) {
 	}
 	return string(b), nil
 }
+
+// ThirdPartyContainerRepo returns the container registry to use for third-party containers.
+// If config registry is set to the Gitpod registry, the third-party registry is returned. If
+// config registry is different, that repository is returned and deployment expected to mirror
+// the images to their registry
+func ThirdPartyContainerRepo(configRegistry string, thirdPartyRegistry string) string {
+	configRegistry = strings.TrimSuffix(configRegistry, "/")
+
+	if configRegistry == GitpodContainerRegistry {
+		return thirdPartyRegistry
+	}
+
+	return configRegistry
+}

installer/pkg/common/constants.go

@@ -15,10 +15,15 @@ const (
 	AppName                     = "gitpod"
 	BlobServeServicePort        = 4000
 	CertManagerCAIssuer         = "ca-issuer"
+	DockerRegistryURL           = "docker.io"
 	DockerRegistryName          = "registry"
+	GitpodContainerRegistry     = "eu.gcr.io/gitpod-core-dev/build"
 	InClusterDbSecret           = "mysql"
 	InClusterMessageQueueName   = "rabbitmq"
 	InClusterMessageQueueTLS    = "messagebus-certificates-secret-core"
+	KubeRBACProxyRepo           = "quay.io"
+	KubeRBACProxyImage          = "brancz/kube-rbac-proxy"
+	KubeRBACProxyTag            = "v0.11.0"
 	MinioServiceAPIPort         = 9000
 	MonitoringChart             = "monitoring"
 	ProxyComponent              = "proxy"

installer/pkg/common/objects.go

@@ -14,6 +14,16 @@ import (
 
 func DefaultServiceAccount(component string) RenderFunc {
 	return func(cfg *RenderContext) ([]runtime.Object, error) {
+		pullSecrets := make([]corev1.LocalObjectReference, 0)
+
+		if len(cfg.Config.ImagePullSecrets) > 0 {
+			for _, i := range cfg.Config.ImagePullSecrets {
+				pullSecrets = append(pullSecrets, corev1.LocalObjectReference{
+					Name: i.Name,
+				})
+			}
+		}
+
 		return []runtime.Object{
 			&corev1.ServiceAccount{
 				TypeMeta: TypeMetaServiceAccount,
@@ -23,6 +33,7 @@ func DefaultServiceAccount(component string) RenderFunc {
 					Labels:    DefaultLabels(component),
 				},
 				AutomountServiceAccountToken: pointer.Bool(true),
+				ImagePullSecrets:             pullSecrets,
 			},
 		}, nil
 	}

installer/pkg/components/agent-smith/daemonset.go

@@ -78,7 +78,7 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 							Privileged: pointer.Bool(true),
 							ProcMount:  func() *corev1.ProcMountType { r := corev1.DefaultProcMount; return &r }(),
 						},
-					}, *common.KubeRBACProxyContainer()},
+					}, *common.KubeRBACProxyContainer(ctx)},
 					Volumes: []corev1.Volume{{
 						Name: "config",
 						VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{

installer/pkg/components/blobserve/deployment.go

@@ -125,7 +125,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								MountPath: "/mnt/pull-secret.json",
 								SubPath:   ".dockerconfigjson",
 							}},
-						}, *common.KubeRBACProxyContainer()},
+						}, *common.KubeRBACProxyContainer(ctx)},
 					},
 				},
 			},

installer/pkg/components/database/incluster/helm.go

@@ -33,6 +33,12 @@ var Helm = common.CompositeHelmFunc(
 					helm.KeyValue("mysql.auth.username", Username),
 					helm.KeyValue("mysql.initdbScriptsConfigMap", SQLInitScripts),
 					helm.KeyValue("mysql.serviceAccount.name", Component),
+					helm.ImagePullSecrets("mysql.image.pullSecrets", cfg),
+					helm.KeyValue("mysql.image.registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)),
+					helm.ImagePullSecrets("mysql.metrics.image.pullSecrets", cfg),
+					helm.KeyValue("mysql.metrics.image.registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)),
+					helm.ImagePullSecrets("mysql.volumePermissions.image.pullSecrets", cfg),
+					helm.KeyValue("mysql.volumePermissions.image.registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)),
 				},
 				// This is too complex to be sent as a string
 				FileValues: []string{

installer/pkg/components/database/init/constants.go

@@ -6,7 +6,7 @@ package init
 
 const (
 	Component       = "dbinit"
-	dbSessionsImage = "mysql"
+	dbSessionsImage = "library/mysql"
 	dbSessionsTag   = "5.7.34"
 	initScriptDir   = "files"
 	sqlInitScripts  = "db-init-scripts"

installer/pkg/components/database/init/job.go

@@ -47,7 +47,7 @@ func job(ctx *common.RenderContext) ([]runtime.Object, error) {
 					InitContainers: []corev1.Container{*common.DatabaseWaiterContainer(ctx)},
 					Containers: []corev1.Container{{
 						Name:            fmt.Sprintf("%s-session", Component),
-						Image:           fmt.Sprintf("%s:%s", dbSessionsImage, dbSessionsTag),
+						Image:           common.ImageName(common.ThirdPartyContainerRepo(ctx.Config.Repository, ""), dbSessionsImage, dbSessionsTag),
 						ImagePullPolicy: corev1.PullIfNotPresent,
 						Env: common.MergeEnv(
 							common.DatabaseEnv(&ctx.Config),

installer/pkg/components/docker-registry/helm.go

@@ -23,11 +23,21 @@ var Helm = common.CompositeHelmFunc(
 			return nil, err
 		}
 
+		repository := fmt.Sprintf("%s/library/registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL))
+
 		registryValues := []string{
 			helm.KeyValue(fmt.Sprintf("docker-registry.podAnnotations.%s", strings.Replace(common.AnnotationConfigChecksum, ".", "\\.", -1)), secretHash),
 			helm.KeyValue("docker-registry.fullnameOverride", RegistryName),
 			helm.KeyValue("docker-registry.service.port", strconv.Itoa(common.ProxyContainerHTTPSPort)),
 			helm.KeyValue("docker-registry.tlsSecretName", BuiltInRegistryCerts),
+			helm.KeyValue("docker-registry.image.repository", repository),
+		}
+
+		if len(cfg.Config.ImagePullSecrets) > 0 {
+			// This chart doesn't add in the "name/value" pair format
+			for k, v := range cfg.Config.ImagePullSecrets {
+				registryValues = append(registryValues, helm.KeyValue(fmt.Sprintf("docker-registry.imagePullSecrets[%d].name", k), v.Name))
+			}
 		}
 
 		inCluster := pointer.BoolDeref(cfg.Config.ContainerRegistry.InCluster, false)

installer/pkg/components/image-builder-mk3/deployment.go

@@ -162,7 +162,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							*common.InternalCAVolumeMount(),
 						},
 					},
-						*common.KubeRBACProxyContainer(),
+						*common.KubeRBACProxyContainer(ctx),
 					},
 				},
 			},

installer/pkg/components/jaeger-operator/helm.go

@@ -5,6 +5,7 @@
 package jaegeroperator
 
 import (
+	"fmt"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/helm"
 	"github.com/gitpod-io/gitpod/installer/third_party/charts"
@@ -14,12 +15,17 @@ import (
 
 var Helm = common.CompositeHelmFunc(
 	helm.ImportTemplate(charts.JaegerOperator(), helm.TemplateConfig{}, func(cfg *common.RenderContext) (*common.HelmConfig, error) {
+		repository := common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)
+		image := "jaegertracing/jaeger-operator"
+
 		return &common.HelmConfig{
 			Enabled: pointer.BoolDeref(cfg.Config.Jaeger.InCluster, false),
 			Values: &values.Options{
 				Values: []string{
 					helm.KeyValue("jaeger-operator.crd.install", "true"),
 					helm.KeyValue("jaeger-operator.rbac.clusterRole", "true"),
+					helm.ImagePullSecrets("jaeger-operator.image.imagePullSecrets", cfg),
+					helm.KeyValue("jaeger-operator.image.repository", fmt.Sprintf("%s/%s", repository, image)),
 				},
 			},
 		}, nil

installer/pkg/components/minio/azure/minio.go

@@ -12,26 +12,29 @@ import (
 	"helm.sh/helm/v3/pkg/cli/values"
 )
 
-var Helm = func(apiPort int32, consolePort int32) common.HelmFunc {
+var Helm = func(apiPort int32, consolePort int32, commonHelmValues []string) common.HelmFunc {
 	return common.CompositeHelmFunc(
 		helm.ImportTemplate(charts.Minio(), helm.TemplateConfig{}, func(cfg *common.RenderContext) (*common.HelmConfig, error) {
 			return &common.HelmConfig{
 				Enabled: true,
 				Values: &values.Options{
-					Values: []string{
-						helm.KeyValue("minio.gateway.enabled", "true"),
-						helm.KeyValue("minio.gateway.auth.azure.accessKey", cfg.Values.StorageAccessKey), // Azure value actually taken from secret - used for console/API access
-						helm.KeyValue("minio.gateway.auth.azure.secretKey", cfg.Values.StorageSecretKey), // Ditto
-						helm.KeyValue("minio.gateway.auth.azure.storageAccountNameExistingSecret", cfg.Config.ObjectStorage.Azure.Credentials.Name),
-						helm.KeyValue("minio.gateway.auth.azure.storageAccountNameExistingSecretKey", "accountName"),
-						helm.KeyValue("minio.gateway.auth.azure.storageAccountKeyExistingSecret", cfg.Config.ObjectStorage.Azure.Credentials.Name),
-						helm.KeyValue("minio.gateway.auth.azure.storageAccountKeyExistingSecretKey", "accountKey"),
-						helm.KeyValue("minio.gateway.replicaCount", "2"),
-						helm.KeyValue("minio.gateway.type", "azure"),
-						helm.KeyValue("minio.persistence.enabled", "false"),
-						helm.KeyValue("minio.service.ports.api", fmt.Sprintf("%d", apiPort)),
-						helm.KeyValue("minio.service.ports.console", fmt.Sprintf("%d", consolePort)),
-					},
+					Values: append(
+						[]string{
+							helm.KeyValue("minio.gateway.enabled", "true"),
+							helm.KeyValue("minio.gateway.auth.azure.accessKey", cfg.Values.StorageAccessKey), // Azure value actually taken from secret - used for console/API access
+							helm.KeyValue("minio.gateway.auth.azure.secretKey", cfg.Values.StorageSecretKey), // Ditto
+							helm.KeyValue("minio.gateway.auth.azure.storageAccountNameExistingSecret", cfg.Config.ObjectStorage.Azure.Credentials.Name),
+							helm.KeyValue("minio.gateway.auth.azure.storageAccountNameExistingSecretKey", "accountName"),
+							helm.KeyValue("minio.gateway.auth.azure.storageAccountKeyExistingSecret", cfg.Config.ObjectStorage.Azure.Credentials.Name),
+							helm.KeyValue("minio.gateway.auth.azure.storageAccountKeyExistingSecretKey", "accountKey"),
+							helm.KeyValue("minio.gateway.replicaCount", "2"),
+							helm.KeyValue("minio.gateway.type", "azure"),
+							helm.KeyValue("minio.persistence.enabled", "false"),
+							helm.KeyValue("minio.service.ports.api", fmt.Sprintf("%d", apiPort)),
+							helm.KeyValue("minio.service.ports.console", fmt.Sprintf("%d", consolePort)),
+						},
+						commonHelmValues...,
+					),
 				},
 			}, nil
 		}),

installer/pkg/components/minio/helm.go

@@ -10,16 +10,24 @@ import (
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/components/minio/azure"
 	"github.com/gitpod-io/gitpod/installer/pkg/components/minio/incluster"
+	"github.com/gitpod-io/gitpod/installer/pkg/helm"
 	"k8s.io/utils/pointer"
 )
 
 var Helm = common.CompositeHelmFunc(
 	func(cfg *common.RenderContext) ([]string, error) {
+		commonHelmValues := []string{
+			helm.ImagePullSecrets("minio.image.pullSecrets", cfg),
+			helm.KeyValue("minio.image.registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)),
+			helm.ImagePullSecrets("minio.volumePermissions.image.pullSecrets", cfg),
+			helm.KeyValue("minio.volumePermissions.image.registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)),
+		}
+
 		if pointer.BoolDeref(cfg.Config.ObjectStorage.InCluster, false) {
-			return incluster.Helm(ServiceAPIPort, ServiceConsolePort)(cfg)
+			return incluster.Helm(ServiceAPIPort, ServiceConsolePort, commonHelmValues)(cfg)
 		}
 		if cfg.Config.ObjectStorage.Azure != nil {
-			return azure.Helm(ServiceAPIPort, ServiceConsolePort)(cfg)
+			return azure.Helm(ServiceAPIPort, ServiceConsolePort, commonHelmValues)(cfg)
 		}
 
 		return nil, nil

installer/pkg/components/minio/incluster/minio.go

@@ -13,7 +13,7 @@ import (
 	"helm.sh/helm/v3/pkg/cli/values"
 )
 
-var Helm = func(apiPort int32, consolePort int32) common.HelmFunc {
+var Helm = func(apiPort int32, consolePort int32, commonHelmValues []string) common.HelmFunc {
 	return common.CompositeHelmFunc(
 		helm.ImportTemplate(charts.Minio(), helm.TemplateConfig{}, func(cfg *common.RenderContext) (*common.HelmConfig, error) {
 			affinity, err := helm.AffinityYaml(cluster.AffinityLabelMeta)
@@ -29,12 +29,15 @@ var Helm = func(apiPort int32, consolePort int32) common.HelmFunc {
 			return &common.HelmConfig{
 				Enabled: true,
 				Values: &values.Options{
-					Values: []string{
-						helm.KeyValue("minio.auth.rootUser", cfg.Values.StorageAccessKey),
-						helm.KeyValue("minio.auth.rootPassword", cfg.Values.StorageSecretKey),
-						helm.KeyValue("minio.service.ports.api", fmt.Sprintf("%d", apiPort)),
-						helm.KeyValue("minio.service.ports.console", fmt.Sprintf("%d", consolePort)),
-					},
+					Values: append(
+						[]string{
+							helm.KeyValue("minio.auth.rootUser", cfg.Values.StorageAccessKey),
+							helm.KeyValue("minio.auth.rootPassword", cfg.Values.StorageSecretKey),
+							helm.KeyValue("minio.service.ports.api", fmt.Sprintf("%d", apiPort)),
+							helm.KeyValue("minio.service.ports.console", fmt.Sprintf("%d", consolePort)),
+						},
+						commonHelmValues...,
+					),
 					// This is too complex to be sent as a string
 					FileValues: []string{
 						affinityTemplate,

installer/pkg/components/proxy/constants.go

@@ -13,8 +13,11 @@ const (
 	ContainerHTTPSPort    = common.ProxyContainerHTTPSPort
 	ContainerHTTPSName    = common.ProxyContainerHTTPSName
 	PrometheusPort        = 9500
-	InitContainerImage    = "alpine:3.15"
-	KubeRBACProxyImage    = "quay.io/brancz/kube-rbac-proxy:v0.11.0"
+	InitContainerImage    = "library/alpine"
+	InitContainerTag      = "3.15"
+	KubeRBACProxyRepo     = common.KubeRBACProxyRepo
+	KubeRBACProxyImage    = common.KubeRBACProxyImage
+	KubeRBACProxyTag      = common.KubeRBACProxyTag
 	MetricsContainerName  = "metrics"
 	ReadinessPort         = 8003
 	RegistryAuthSecret    = common.RegistryAuthSecret

installer/pkg/components/proxy/deployment.go

@@ -129,7 +129,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						Volumes: volumes,
 						InitContainers: []corev1.Container{{
 							Name:            "sysctl",
-							Image:           InitContainerImage,
+							Image:           common.ImageName(common.ThirdPartyContainerRepo(ctx.Config.Repository, common.DockerRegistryURL), InitContainerImage, InitContainerTag),
 							ImagePullPolicy: corev1.PullIfNotPresent,
 							SecurityContext: &corev1.SecurityContext{
 								Privileged: pointer.Bool(true),
@@ -142,7 +142,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						}},
 						Containers: []corev1.Container{{
 							Name:            "kube-rbac-proxy",
-							Image:           KubeRBACProxyImage,
+							Image:           common.ImageName(common.ThirdPartyContainerRepo(ctx.Config.Repository, KubeRBACProxyRepo), KubeRBACProxyImage, KubeRBACProxyTag),
 							ImagePullPolicy: corev1.PullIfNotPresent,
 							Args: []string{
 								"--v=10",

installer/pkg/components/rabbitmq/helm.go

@@ -279,6 +279,10 @@ var Helm = common.CompositeHelmFunc(
 					helm.KeyValue("rabbitmq.auth.tls.existingSecret", TLSSecret),
 					helm.KeyValue(fmt.Sprintf("rabbitmq.extraSecrets.%s.username", InClusterDbSecret), username),
 					helm.KeyValue(fmt.Sprintf("rabbitmq.extraSecrets.%s.password", InClusterDbSecret), password),
+					helm.ImagePullSecrets("rabbitmq.image.pullSecrets", cfg),
+					helm.KeyValue("rabbitmq.image.registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)),
+					helm.ImagePullSecrets("volumePermissions.image.pullSecrets", cfg),
+					helm.KeyValue("rabbitmq.volumePermissions.image.registry", common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)),
 				},
 				// This is too complex to be sent as a string
 				FileValues: []string{

installer/pkg/components/registry-facade/daemonset.go

@@ -176,7 +176,7 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 						}, volumeMounts...),
 					},
 
-						*common.KubeRBACProxyContainer(),
+						*common.KubeRBACProxyContainer(ctx),
 					},
 					Volumes: append([]corev1.Volume{{
 						Name:         "cache",

installer/pkg/components/server/deployment.go

@@ -149,7 +149,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								MountPath: "/ws-manager-client-tls-certs",
 								ReadOnly:  true,
 							}},
-						}, *common.KubeRBACProxyContainer()},
+						}, *common.KubeRBACProxyContainer(ctx)},
 					},
 				},
 			},

installer/pkg/components/ws-daemon/daemonset.go

@@ -6,7 +6,6 @@ package wsdaemon
 
 import (
 	"fmt"
-
 	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
@@ -32,7 +31,7 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 	initContainers := []corev1.Container{
 		{
 			Name:  "disable-kube-health-monitor",
-			Image: "ubuntu:20.04",
+			Image: common.ImageName(common.ThirdPartyContainerRepo(ctx.Config.Repository, common.DockerRegistryURL), "library/ubuntu", "20.04"),
 			Command: []string{
 				"/usr/bin/nsenter",
 				"-t",
@@ -281,7 +280,7 @@ fi
 					Privileged: pointer.Bool(true),
 				},
 			},
-			*common.KubeRBACProxyContainer(),
+			*common.KubeRBACProxyContainer(ctx),
 		},
 		RestartPolicy:                 "Always",
 		TerminationGracePeriodSeconds: pointer.Int64(30),

installer/pkg/components/ws-manager-bridge/deployment.go

@@ -121,7 +121,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								MountPath: "/ws-manager-client-tls-certs",
 								ReadOnly:  true,
 							}},
-						}, *common.KubeRBACProxyContainer()},
+						}, *common.KubeRBACProxyContainer(ctx)},
 					},
 				},
 			},

installer/pkg/components/ws-scheduler/deployment.go

@@ -96,7 +96,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							MountPath: "/ws-manager-client-tls-certs",
 							ReadOnly:  true,
 						}},
-					}, *common.KubeRBACProxyContainer()},
+					}, *common.KubeRBACProxyContainer(ctx)},
 				},
 			},
 		},