Merge pull request #420 from mfojtik/remove-kco-render

replace kube-core rendering with openshift operators render

pkg/asset/ignition/bootstrap/bootstrap.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/template"
 
@@ -39,8 +40,6 @@ type bootstrapTemplateData struct {
 	EtcdCertSignerImage string
 	EtcdCluster         string
 	EtcdctlImage        string
-	HyperkubeImage      string
-	KubeCoreRenderImage string
 	ReleaseImage        string
 }
 
@@ -75,7 +74,6 @@ func (a *Bootstrap) Dependencies() []asset.Asset {
 		&kubeconfig.Kubelet{},
 		&manifests.Manifests{},
 		&manifests.Tectonic{},
-		&manifests.KubeCoreOperator{},
 	}
 }
 
@@ -97,6 +95,7 @@ func (a *Bootstrap) Generate(dependencies asset.Parents) error {
 
 	a.addBootstrapFiles(dependencies)
 	a.addBootkubeFiles(dependencies, templateData)
+	a.addTemporaryBootkubeFiles(templateData)
 	a.addTectonicFiles(dependencies, templateData)
 	a.addTLSCertFiles(dependencies)
 
@@ -160,37 +159,31 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootst
 		CloudProvider:       getCloudProvider(installConfig),
 		CloudProviderConfig: getCloudProviderConfig(installConfig),
 		DebugConfig:         "",
-		KubeCoreRenderImage: "quay.io/coreos/kube-core-renderer-dev:375423a332f2c12b79438fc6a6da6e448e28ec0f",
 		EtcdCertSignerImage: "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6",
 		EtcdctlImage:        "quay.io/coreos/etcd:v3.2.14",
 		BootkubeImage:       "quay.io/coreos/bootkube:v0.10.0",
 		ReleaseImage:        releaseImage,
-		HyperkubeImage:      "openshift/origin-node:latest",
 		EtcdCluster:         strings.Join(etcdEndpoints, ","),
 	}, nil
 }
 
 func (a *Bootstrap) addBootstrapFiles(dependencies asset.Parents) {
 	kubeletKubeconfig := &kubeconfig.Kubelet{}
-	kubeCoreOperator := &manifests.KubeCoreOperator{}
-	dependencies.Get(kubeletKubeconfig, kubeCoreOperator)
+	dependencies.Get(kubeletKubeconfig)
 
 	a.Config.Storage.Files = append(
 		a.Config.Storage.Files,
 		ignition.FileFromBytes("/etc/kubernetes/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data),
 		ignition.FileFromBytes("/var/lib/kubelet/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data),
 	)
-	a.Config.Storage.Files = append(
-		a.Config.Storage.Files,
-		ignition.FilesFromAsset(rootDir, 0644, kubeCoreOperator)...,
-	)
 	a.Config.Storage.Files = append(
 		a.Config.Storage.Files,
 		ignition.FileFromString("/opt/tectonic/report-progress.sh", 0555, content.ReportShFileContents),
 	)
 }
 
 func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *bootstrapTemplateData) {
+	bootkubeConfigOverridesDir := filepath.Join(rootDir, "bootkube-config-overrides")
 	adminKubeconfig := &kubeconfig.Admin{}
 	manifests := &manifests.Manifests{}
 	dependencies.Get(adminKubeconfig, manifests)
@@ -199,6 +192,12 @@ func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *b
 		a.Config.Storage.Files,
 		ignition.FileFromString("/opt/tectonic/bootkube.sh", 0555, applyTemplateData(content.BootkubeShFileTemplate, templateData)),
 	)
+	for _, o := range content.BootkubeConfigOverrides {
+		a.Config.Storage.Files = append(
+			a.Config.Storage.Files,
+			ignition.FileFromString(filepath.Join(bootkubeConfigOverridesDir, o.Name()), 0600, applyTemplateData(o, templateData)),
+		)
+	}
 	a.Config.Storage.Files = append(
 		a.Config.Storage.Files,
 		ignition.FilesFromAsset(rootDir, 0600, adminKubeconfig)...,
@@ -209,6 +208,36 @@ func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *b
 	)
 }
 
+func (a *Bootstrap) addTemporaryBootkubeFiles(templateData *bootstrapTemplateData) {
+	podCheckpointerBootstrapDir := filepath.Join(rootDir, "pod-checkpointer-operator-bootstrap")
+	for name, data := range content.PodCheckpointerBootkubeManifests {
+		a.Config.Storage.Files = append(
+			a.Config.Storage.Files,
+			ignition.FileFromString(filepath.Join(podCheckpointerBootstrapDir, name), 0644, data),
+		)
+	}
+
+	kubeProxyBootstrapDir := filepath.Join(rootDir, "kube-proxy-operator-bootstrap")
+	for name, data := range content.KubeProxyBootkubeManifests {
+		a.Config.Storage.Files = append(
+			a.Config.Storage.Files,
+			ignition.FileFromString(filepath.Join(kubeProxyBootstrapDir, name), 0644, data),
+		)
+	}
+
+	kubeDNSBootstrapDir := filepath.Join(rootDir, "kube-dns-operator-bootstrap")
+	for name, data := range content.KubeDNSBootkubeManifests {
+		a.Config.Storage.Files = append(
+			a.Config.Storage.Files,
+			ignition.FileFromString(filepath.Join(kubeDNSBootstrapDir, name), 0644, data),
+		)
+	}
+	a.Config.Storage.Files = append(
+		a.Config.Storage.Files,
+		ignition.FileFromString(filepath.Join(kubeDNSBootstrapDir, "kube-dns-svc.yaml"), 0644, applyTemplateData(content.BootkubeKubeDNSService, templateData)),
+	)
+}
+
 func (a *Bootstrap) addTectonicFiles(dependencies asset.Parents, templateData *bootstrapTemplateData) {
 	tectonic := &manifests.Tectonic{}
 	dependencies.Get(tectonic)

pkg/asset/ignition/bootstrap/content/bootkube.go

@@ -31,7 +31,7 @@ var (
 	BootkubeShFileTemplate = template.Must(template.New("bootkube.sh").Parse(`#!/usr/bin/env bash
 set -e
 
-mkdir --parents /etc/kubernetes/manifests/
+mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests}
 
 MACHINE_CONFIG_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image machine-config-operator)
 MACHINE_CONFIG_CONTROLLER_IMAGE=$(podman run --rm {{.ReleaseImage}} image machine-config-controller)
@@ -42,6 +42,9 @@ KUBE_APISERVER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster-
 KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster-kube-controller-manager-operator)
 KUBE_SCHEDULER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster-kube-scheduler-operator)
 
+OPENSHIFT_HYPERSHIFT_IMAGE=$(podman run --rm {{.ReleaseImage}} image hypershift)
+OPENSHIFT_HYPERKUBE_IMAGE=$(podman run --rm {{.ReleaseImage}} image hyperkube)
+
 if [ ! -d cvo-bootstrap ]
 then
 	echo "Rendering Cluster Version Operator Manifests..."
@@ -57,22 +60,7 @@ then
 	cp --recursive cvo-bootstrap/manifests .
 fi
 
-if [ ! -d kco-bootstrap ]
-then
-	echo "Rendering Kubernetes core manifests..."
-
-	# shellcheck disable=SC2154
-	podman run \
-		--volume "$PWD:/assets:z" \
-		--volume /etc/kubernetes:/etc/kubernetes:z \
-		"{{.KubeCoreRenderImage}}" \
-		--config=/assets/kco-config.yaml \
-		--output=/assets/kco-bootstrap
-
-	cp --recursive kco-bootstrap/bootstrap-configs /etc/kubernetes/bootstrap-configs
-	cp --recursive kco-bootstrap/bootstrap-manifests .
-	cp --recursive kco-bootstrap/manifests .
-fi
+mkdir --parents ./{bootstrap-manifests,manifests}
 
 if [ ! -d kube-apiserver-bootstrap ]
 then
@@ -84,14 +72,16 @@ then
 		"${KUBE_APISERVER_OPERATOR_IMAGE}" \
 		/usr/bin/cluster-kube-apiserver-operator render \
 		--manifest-etcd-serving-ca=etcd-client-ca.crt \
+		--manifest-etcd-server-urls={{.EtcdCluster}} \
+		--manifest-image=${OPENSHIFT_HYPERSHIFT_IMAGE} \
 		--asset-input-dir=/assets/tls \
 		--asset-output-dir=/assets/kube-apiserver-bootstrap \
-		--config-output-file=/assets/kube-apiserver-bootstrap/config
+		--config-output-file=/assets/kube-apiserver-bootstrap/config \
+		--config-override-files=/assets/bootkube-config-overrides/kube-apiserver-config-overrides.yaml
 
-	# TODO: copy the bootstrap manifests to replace kube-core-operator
-	cp --recursive kube-apiserver-bootstrap/manifests/00_openshift-kube-apiserver-ns.yaml manifests/00_openshift-kube-apiserver-ns.yaml
-	cp --recursive kube-apiserver-bootstrap/manifests/secret-* manifests/
-	cp --recursive kube-apiserver-bootstrap/manifests/configmap-* manifests/
+	cp kube-apiserver-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-apiserver-config.yaml
+	cp kube-apiserver-bootstrap/bootstrap-manifests/* bootstrap-manifests/
+	cp kube-apiserver-bootstrap/manifests/* manifests/
 fi
 
 if [ ! -d kube-controller-manager-bootstrap ]
@@ -103,13 +93,15 @@ then
 		--volume "$PWD:/assets:z" \
 		"${KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE}" \
 		/usr/bin/cluster-kube-controller-manager-operator render \
+		--manifest-image=${OPENSHIFT_HYPERKUBE_IMAGE} \
 		--asset-input-dir=/assets/tls \
 		--asset-output-dir=/assets/kube-controller-manager-bootstrap \
-		--config-output-file=/assets/kube-controller-manager-bootstrap/config
+		--config-output-file=/assets/kube-controller-manager-bootstrap/config \
+		--config-override-files=/assets/bootkube-config-overrides/kube-controller-manager-config-overrides.yaml
 
-	# TODO: copy the bootstrap manifests to replace kube-core-operator
-	cp --recursive kube-controller-manager-bootstrap/manifests/00_openshift-kube-controller-manager-ns.yaml manifests/00_openshift-kube-controller-manager-ns.yaml
-	cp --recursive kube-controller-manager-bootstrap/manifests/configmap-* manifests/
+	cp kube-controller-manager-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-controller-manager-config.yaml
+	cp kube-controller-manager-bootstrap/bootstrap-manifests/* bootstrap-manifests/
+	cp kube-controller-manager-bootstrap/manifests/* manifests/
 fi
 
 if [ ! -d kube-scheduler-bootstrap ]
@@ -121,15 +113,22 @@ then
 		--volume "$PWD:/assets:z" \
 		"${KUBE_SCHEDULER_OPERATOR_IMAGE}" \
 		/usr/bin/cluster-kube-scheduler-operator render \
+		--manifest-image=${OPENSHIFT_HYPERKUBE_IMAGE} \
 		--asset-input-dir=/assets/tls \
 		--asset-output-dir=/assets/kube-scheduler-bootstrap \
 		--config-output-file=/assets/kube-scheduler-bootstrap/config
 
-	# TODO: copy the bootstrap manifests to replace kube-core-operator
-	cp --recursive kube-scheduler-bootstrap/manifests/00_openshift-kube-scheduler-ns.yaml manifests/00_openshift-kube-scheduler-ns.yaml
-	cp --recursive kube-scheduler-bootstrap/manifests/configmap-* manifests/
+	cp kube-scheduler-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-scheduler-config.yaml
+	cp kube-scheduler-bootstrap/bootstrap-manifests/* bootstrap-manifests/
+	cp kube-scheduler-bootstrap/manifests/* manifests/
 fi
 
+# TODO: Remove this when checkpointer, kube-proxy and kube-dns are properly rendered by corresponding operators.
+echo "Installing temporary bootstrap manifests..."
+cp pod-checkpointer-operator-bootstrap/* manifests/
+cp kube-proxy-operator-bootstrap/* manifests/
+cp kube-dns-operator-bootstrap/* manifests/
+
 if [ ! -d mco-bootstrap ]
 then
 	echo "Rendering MCO manifests..."
@@ -152,8 +151,8 @@ then
 	# 1. read the controller config rendered by MachineConfigOperator
 	# 2. read the default MachineConfigPools rendered by MachineConfigOperator
 	# 3. read any additional MachineConfigs that are needed for the default MachineConfigPools.
-	mkdir --parents /etc/mcc/bootstrap/
-	cp --recursive mco-bootstrap/manifests /etc/mcc/bootstrap/manifests
+	mkdir --parents /etc/mcc/bootstrap/manifests /etc/kubernetes/manifests/
+	cp mco-bootstrap/manifests/* /etc/mcc/bootstrap/manifests/
 	cp mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/
 
 	# /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets.
@@ -232,3 +231,30 @@ podman run \
 	start --asset-dir=/assets
 `))
 )
+
+var (
+	// BootkubeConfigOverrides contains the configuration override files passed to the render commands of the components.
+	// These are supposed to be customized by the installer where the config differs from the operator render default.
+	BootkubeConfigOverrides = []*template.Template{
+		KubeApiserverConfigOverridesTemplate,
+		KubeControllerManagerConfigOverridesTemplate,
+	}
+)
+
+var (
+	// KubeApiserverConfigOverridesTemplate are overrides that the installer passes to the default config of the
+	// kube-apiserver rendered by the cluster-kube-apiserver-operator.
+	KubeApiserverConfigOverridesTemplate = template.Must(template.New("kube-apiserver-config-overrides.yaml").Parse(`
+apiVersion: kubecontrolplane.config.openshift.io/v1
+kind: KubeAPIServerConfig
+kubeletClientInfo:
+  ca: "" # kubelet uses self-signed serving certs. TODO: fix kubelet pki
+`))
+
+	// KubeControllerManagerConfigOverridesTemplate are overrides that the installer passes to the default config of the
+	// kube-controller-manager rendered by the cluster-kube-controller-manager-operator.
+	KubeControllerManagerConfigOverridesTemplate = template.Must(template.New("kube-controller-manager-config-overrides.yaml").Parse(`
+apiVersion: kubecontrolplane.config.openshift.io/v1
+kind: KubeControllerManagerConfig
+`))
+)