Merge pull request #879 from abhinavdahiya/master_bootstrapping

 reduce permissions on bootstrap kubeconfig used by masters

data/data/bootstrap/files/usr/local/bin/bootkube.sh.template

@@ -46,6 +46,8 @@ then
 
 	cp cvo-bootstrap/bootstrap/* bootstrap-manifests/
 	cp cvo-bootstrap/manifests/* manifests/
+	## FIXME: CVO should use `/etc/kubernetes/bootstrap-secrets/kubeconfig` instead
+	cp auth/kubeconfig /etc/kubernetes/kubeconfig
 fi
 
 if [ ! -d kube-apiserver-bootstrap ]
@@ -136,9 +138,12 @@ then
 	# 1. read the controller config rendered by MachineConfigOperator
 	# 2. read the default MachineConfigPools rendered by MachineConfigOperator
 	# 3. read any additional MachineConfigs that are needed for the default MachineConfigPools.
-	mkdir --parents /etc/mcc/bootstrap/manifests /etc/kubernetes/manifests/
-	cp mco-bootstrap/manifests/* /etc/mcc/bootstrap/manifests/
-	cp mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/
+	mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests
+	cp mco-bootstrap/bootstrap/manifests/* /etc/mcc/bootstrap/
+	cp openshift/* /etc/mcc/bootstrap/
+	cp auth/kubeconfig-kubelet /etc/mcs/kubeconfig
+	cp mco-bootstrap/bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/
+	cp mco-bootstrap/manifests/* manifests/
 
 	# /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets.
 	mkdir --parents /etc/ssl/mcs/

data/data/bootstrap/systemd/units/kubelet.service.template

@@ -5,7 +5,6 @@ Wants=rpc-statd.service
 [Service]
 Type=notify
 ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests
-ExecStartPre=/usr/bin/bash -c "gawk '/certificate-authority-data/ {print $2}' /etc/kubernetes/kubeconfig | base64 --decode > /etc/kubernetes/ca.crt"
 Environment=KUBELET_RUNTIME_REQUEST_TIMEOUT=10m
 EnvironmentFile=-/etc/kubernetes/kubelet-env
 
@@ -18,8 +17,6 @@ ExecStart=/usr/bin/hyperkube \
     --allow-privileged \
     --minimum-container-ttl-duration=6m0s \
     --cluster-domain=cluster.local \
-    --client-ca-file=/etc/kubernetes/ca.crt \
-    --anonymous-auth=false \
     --cgroup-driver=systemd \
     --serialize-image-pulls=false \
     --v=2 \

pkg/asset/ignition/bootstrap/bootstrap.go

@@ -2,7 +2,6 @@ package bootstrap
 
 import (
 	"bytes"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -40,12 +39,11 @@ const (
 // bootstrapTemplateData is the data to use to replace values in bootstrap
 // template files.
 type bootstrapTemplateData struct {
-	EtcdCertSignerImage   string
-	EtcdCluster           string
-	EtcdctlImage          string
-	PullSecret            string
-	ReleaseImage          string
-	AdminKubeConfigBase64 string
+	EtcdCertSignerImage string
+	EtcdCluster         string
+	EtcdctlImage        string
+	PullSecret          string
+	ReleaseImage        string
 }
 
 // Bootstrap is an asset that generates the ignition config for bootstrap nodes.
@@ -82,10 +80,9 @@ func (a *Bootstrap) Dependencies() []asset.Asset {
 // Generate generates the ignition config for the Bootstrap asset.
 func (a *Bootstrap) Generate(dependencies asset.Parents) error {
 	installConfig := &installconfig.InstallConfig{}
-	adminKubeConfig := &kubeconfig.Admin{}
-	dependencies.Get(installConfig, adminKubeConfig)
+	dependencies.Get(installConfig)
 
-	templateData, err := a.getTemplateData(installConfig.Config, adminKubeConfig.File.Data)
+	templateData, err := a.getTemplateData(installConfig.Config)
 	if err != nil {
 		return errors.Wrap(err, "failed to get bootstrap templates")
 	}
@@ -137,7 +134,7 @@ func (a *Bootstrap) Files() []*asset.File {
 }
 
 // getTemplateData returns the data to use to execute bootstrap templates.
-func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, adminKubeConfig []byte) (*bootstrapTemplateData, error) {
+func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootstrapTemplateData, error) {
 	etcdEndpoints := make([]string, installConfig.MasterCount())
 	for i := range etcdEndpoints {
 		etcdEndpoints[i] = fmt.Sprintf("https://%s-etcd-%d.%s:2379", installConfig.ObjectMeta.Name, i, installConfig.BaseDomain)
@@ -150,12 +147,11 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, adminKub
 	}
 
 	return &bootstrapTemplateData{
-		EtcdCertSignerImage:   etcdCertSignerImage,
-		EtcdctlImage:          etcdctlImage,
-		PullSecret:            installConfig.PullSecret,
-		ReleaseImage:          releaseImage,
-		EtcdCluster:           strings.Join(etcdEndpoints, ","),
-		AdminKubeConfigBase64: base64.StdEncoding.EncodeToString(adminKubeConfig),
+		EtcdCertSignerImage: etcdCertSignerImage,
+		EtcdctlImage:        etcdctlImage,
+		PullSecret:          installConfig.PullSecret,
+		ReleaseImage:        releaseImage,
+		EtcdCluster:         strings.Join(etcdEndpoints, ","),
 	}, nil
 }
 
@@ -281,21 +277,10 @@ func readFile(name string, reader io.Reader, templateData interface{}) (finalNam
 }
 
 func (a *Bootstrap) addParentFiles(dependencies asset.Parents) {
-	adminKubeconfig := &kubeconfig.Admin{}
-	kubeletKubeconfig := &kubeconfig.Kubelet{}
 	mfsts := &manifests.Manifests{}
 	openshiftManifests := &manifests.Openshift{}
-	dependencies.Get(adminKubeconfig, kubeletKubeconfig, mfsts, openshiftManifests)
+	dependencies.Get(mfsts, openshiftManifests)
 
-	a.Config.Storage.Files = append(
-		a.Config.Storage.Files,
-		ignition.FilesFromAsset(rootDir, 0600, adminKubeconfig)...,
-	)
-	a.Config.Storage.Files = append(
-		a.Config.Storage.Files,
-		ignition.FileFromBytes("/etc/kubernetes/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data),
-		ignition.FileFromBytes("/var/lib/kubelet/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data),
-	)
 	a.Config.Storage.Files = append(
 		a.Config.Storage.Files,
 		ignition.FilesFromAsset(rootDir, 0644, mfsts)...,
@@ -306,6 +291,8 @@ func (a *Bootstrap) addParentFiles(dependencies asset.Parents) {
 	)
 
 	for _, asset := range []asset.WritableAsset{
+		&kubeconfig.Admin{},
+		&kubeconfig.Kubelet{},
 		&tls.RootCA{},
 		&tls.KubeCA{},
 		&tls.AggregatorCA{},
@@ -322,13 +309,6 @@ func (a *Bootstrap) addParentFiles(dependencies asset.Parents) {
 		dependencies.Get(asset)
 		a.Config.Storage.Files = append(a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, 0600, asset)...)
 	}
-
-	etcdClientCertKey := &tls.EtcdClientCertKey{}
-	dependencies.Get(etcdClientCertKey)
-	a.Config.Storage.Files = append(
-		a.Config.Storage.Files,
-		ignition.FileFromBytes("/etc/ssl/etcd/ca.crt", 0600, etcdClientCertKey.Cert()),
-	)
 }
 
 func applyTemplateData(template *template.Template, templateData interface{}) string {

pkg/asset/tls/kubeletcertkey.go

@@ -29,9 +29,7 @@ func (a *KubeletCertKey) Generate(dependencies asset.Parents) error {
 	dependencies.Get(kubeCA)
 
 	cfg := &CertCfg{
-		// system:masters is a hack to get the kubelet up without kube-core
-		// TODO(node): make kubelet bootstrapping secure with minimal permissions eventually switching to system:node:* CommonName
-		Subject:      pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system", "system:masters"}},
+		Subject:      pkix.Name{CommonName: "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper", Organization: []string{"system:serviceaccounts:openshift-machine-config-operator"}},
 		KeyUsages:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
 		Validity:     ValidityThirtyMinutes,