From 8b5fb5dd8542654b9e4856ab5f13c24a71dca71d Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Tue, 11 Dec 2018 15:43:09 -0800 Subject: [PATCH 1/2] reduce permissions on bootstrap kubeconfig used by masters --- data/data/bootstrap/systemd/units/kubelet.service.template | 3 --- pkg/asset/tls/kubeletcertkey.go | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/data/data/bootstrap/systemd/units/kubelet.service.template b/data/data/bootstrap/systemd/units/kubelet.service.template index ba875583c47..0ac9bb828d9 100644 --- a/data/data/bootstrap/systemd/units/kubelet.service.template +++ b/data/data/bootstrap/systemd/units/kubelet.service.template @@ -5,7 +5,6 @@ Wants=rpc-statd.service [Service] Type=notify ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests -ExecStartPre=/usr/bin/bash -c "gawk '/certificate-authority-data/ {print $2}' /etc/kubernetes/kubeconfig | base64 --decode > /etc/kubernetes/ca.crt" Environment=KUBELET_RUNTIME_REQUEST_TIMEOUT=10m EnvironmentFile=-/etc/kubernetes/kubelet-env @@ -18,8 +17,6 @@ ExecStart=/usr/bin/hyperkube \ --allow-privileged \ --minimum-container-ttl-duration=6m0s \ --cluster-domain=cluster.local \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ --cgroup-driver=systemd \ --serialize-image-pulls=false \ --v=2 \ diff --git a/pkg/asset/tls/kubeletcertkey.go b/pkg/asset/tls/kubeletcertkey.go index b8632e2025e..7da2c4cd28e 100644 --- a/pkg/asset/tls/kubeletcertkey.go +++ b/pkg/asset/tls/kubeletcertkey.go @@ -29,9 +29,7 @@ func (a *KubeletCertKey) Generate(dependencies asset.Parents) error { dependencies.Get(kubeCA) cfg := &CertCfg{ - // system:masters is a hack to get the kubelet up without kube-core - // TODO(node): make kubelet bootstrapping secure with minimal permissions eventually switching to system:node:* CommonName - Subject: pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system", "system:masters"}}, + Subject: pkix.Name{CommonName: "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper", Organization: []string{"system:serviceaccounts:openshift-machine-config-operator"}}, KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, Validity: ValidityThirtyMinutes, From 6da1973fb00a862466c5eb2649c96ce9fa679498 Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Tue, 11 Dec 2018 15:46:57 -0800 Subject: [PATCH 2/2] bootstrap: change mco boostrapping https://github.com/openshift/machine-config-operator/pull/226 sets up all the `manifests` required to approve CSRs for masters --- .../files/usr/local/bin/bootkube.sh.template | 11 ++-- pkg/asset/ignition/bootstrap/bootstrap.go | 52 ++++++------------- 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 2fbdbea1958..67bfdf42d10 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -46,6 +46,8 @@ then cp cvo-bootstrap/bootstrap/* bootstrap-manifests/ cp cvo-bootstrap/manifests/* manifests/ + ## FIXME: CVO should use `/etc/kubernetes/bootstrap-secrets/kubeconfig` instead + cp auth/kubeconfig /etc/kubernetes/kubeconfig fi if [ ! -d kube-apiserver-bootstrap ] @@ -136,9 +138,12 @@ then # 1. read the controller config rendered by MachineConfigOperator # 2. read the default MachineConfigPools rendered by MachineConfigOperator # 3. read any additional MachineConfigs that are needed for the default MachineConfigPools. - mkdir --parents /etc/mcc/bootstrap/manifests /etc/kubernetes/manifests/ - cp mco-bootstrap/manifests/* /etc/mcc/bootstrap/manifests/ - cp mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/ + mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests + cp mco-bootstrap/bootstrap/manifests/* /etc/mcc/bootstrap/ + cp openshift/* /etc/mcc/bootstrap/ + cp auth/kubeconfig-kubelet /etc/mcs/kubeconfig + cp mco-bootstrap/bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/ + cp mco-bootstrap/manifests/* manifests/ # /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets. mkdir --parents /etc/ssl/mcs/ diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index b2de22554a3..977eff061af 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -2,7 +2,6 @@ package bootstrap import ( "bytes" - "encoding/base64" "encoding/json" "fmt" "io" @@ -40,12 +39,11 @@ const ( // bootstrapTemplateData is the data to use to replace values in bootstrap // template files. type bootstrapTemplateData struct { - EtcdCertSignerImage string - EtcdCluster string - EtcdctlImage string - PullSecret string - ReleaseImage string - AdminKubeConfigBase64 string + EtcdCertSignerImage string + EtcdCluster string + EtcdctlImage string + PullSecret string + ReleaseImage string } // Bootstrap is an asset that generates the ignition config for bootstrap nodes. @@ -82,10 +80,9 @@ func (a *Bootstrap) Dependencies() []asset.Asset { // Generate generates the ignition config for the Bootstrap asset. func (a *Bootstrap) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} - adminKubeConfig := &kubeconfig.Admin{} - dependencies.Get(installConfig, adminKubeConfig) + dependencies.Get(installConfig) - templateData, err := a.getTemplateData(installConfig.Config, adminKubeConfig.File.Data) + templateData, err := a.getTemplateData(installConfig.Config) if err != nil { return errors.Wrap(err, "failed to get bootstrap templates") } @@ -137,7 +134,7 @@ func (a *Bootstrap) Files() []*asset.File { } // getTemplateData returns the data to use to execute bootstrap templates. -func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, adminKubeConfig []byte) (*bootstrapTemplateData, error) { +func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootstrapTemplateData, error) { etcdEndpoints := make([]string, installConfig.MasterCount()) for i := range etcdEndpoints { etcdEndpoints[i] = fmt.Sprintf("https://%s-etcd-%d.%s:2379", installConfig.ObjectMeta.Name, i, installConfig.BaseDomain) @@ -150,12 +147,11 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, adminKub } return &bootstrapTemplateData{ - EtcdCertSignerImage: etcdCertSignerImage, - EtcdctlImage: etcdctlImage, - PullSecret: installConfig.PullSecret, - ReleaseImage: releaseImage, - EtcdCluster: strings.Join(etcdEndpoints, ","), - AdminKubeConfigBase64: base64.StdEncoding.EncodeToString(adminKubeConfig), + EtcdCertSignerImage: etcdCertSignerImage, + EtcdctlImage: etcdctlImage, + PullSecret: installConfig.PullSecret, + ReleaseImage: releaseImage, + EtcdCluster: strings.Join(etcdEndpoints, ","), }, nil } @@ -281,21 +277,10 @@ func readFile(name string, reader io.Reader, templateData interface{}) (finalNam } func (a *Bootstrap) addParentFiles(dependencies asset.Parents) { - adminKubeconfig := &kubeconfig.Admin{} - kubeletKubeconfig := &kubeconfig.Kubelet{} mfsts := &manifests.Manifests{} openshiftManifests := &manifests.Openshift{} - dependencies.Get(adminKubeconfig, kubeletKubeconfig, mfsts, openshiftManifests) + dependencies.Get(mfsts, openshiftManifests) - a.Config.Storage.Files = append( - a.Config.Storage.Files, - ignition.FilesFromAsset(rootDir, 0600, adminKubeconfig)..., - ) - a.Config.Storage.Files = append( - a.Config.Storage.Files, - ignition.FileFromBytes("/etc/kubernetes/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data), - ignition.FileFromBytes("/var/lib/kubelet/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data), - ) a.Config.Storage.Files = append( a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, 0644, mfsts)..., @@ -306,6 +291,8 @@ func (a *Bootstrap) addParentFiles(dependencies asset.Parents) { ) for _, asset := range []asset.WritableAsset{ + &kubeconfig.Admin{}, + &kubeconfig.Kubelet{}, &tls.RootCA{}, &tls.KubeCA{}, &tls.AggregatorCA{}, @@ -322,13 +309,6 @@ func (a *Bootstrap) addParentFiles(dependencies asset.Parents) { dependencies.Get(asset) a.Config.Storage.Files = append(a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, 0600, asset)...) } - - etcdClientCertKey := &tls.EtcdClientCertKey{} - dependencies.Get(etcdClientCertKey) - a.Config.Storage.Files = append( - a.Config.Storage.Files, - ignition.FileFromBytes("/etc/ssl/etcd/ca.crt", 0600, etcdClientCertKey.Cert()), - ) } func applyTemplateData(template *template.Template, templateData interface{}) string {