From 8865eb720465a261d0be25c67a0f021ed1b54d86 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Mon, 4 Mar 2019 12:30:20 -0500 Subject: [PATCH 1/2] data/data/*: add configs to support etcd metrics proxy Signed-off-by: Sam Batschelet --- data/data/manifests/bootkube/etcd-service.yaml | 3 +++ ...onfig-configmap-etcd-metrics-serving-ca.yaml.template | 8 ++++++++ ...shift-config-secret-etcd-metrics-client.yaml.template | 9 +++++++++ 3 files changed, 20 insertions(+) create mode 100644 data/data/manifests/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.yaml.template create mode 100644 data/data/manifests/bootkube/openshift-config-secret-etcd-metrics-client.yaml.template diff --git a/data/data/manifests/bootkube/etcd-service.yaml b/data/data/manifests/bootkube/etcd-service.yaml index 9a24566801a..d2ca703e4e4 100644 --- a/data/data/manifests/bootkube/etcd-service.yaml +++ b/data/data/manifests/bootkube/etcd-service.yaml @@ -13,3 +13,6 @@ spec: - name: etcd port: 2379 protocol: TCP + - name: etcd-metrics + port: 9979 + protocol: TCP diff --git a/data/data/manifests/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.yaml.template b/data/data/manifests/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.yaml.template new file mode 100644 index 00000000000..06230707ce6 --- /dev/null +++ b/data/data/manifests/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.yaml.template @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: etcd-metrics-serving-ca + namespace: openshift-config +data: + ca-bundle.crt: | + {{.EtcdMetricsCaCert | indent 4}} diff --git a/data/data/manifests/bootkube/openshift-config-secret-etcd-metrics-client.yaml.template b/data/data/manifests/bootkube/openshift-config-secret-etcd-metrics-client.yaml.template new file mode 100644 index 00000000000..1baa05238f8 --- /dev/null +++ b/data/data/manifests/bootkube/openshift-config-secret-etcd-metrics-client.yaml.template @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: etcd-metrics-client + namespace: openshift-config +type: SecretTypeTLS +data: + tls.crt: {{ .EtcdMetricsClientCert }} + tls.key: {{ .EtcdMetricsClientKey }} From cb4bbecdfedebc1c1b2f880f44388019a969bd03 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Mon, 4 Mar 2019 12:30:34 -0500 Subject: [PATCH 2/2] pkg/asset/*: add etcd metrics TLS assets Signed-off-by: Sam Batschelet --- pkg/asset/ignition/bootstrap/bootstrap.go | 6 + pkg/asset/manifests/operators.go | 33 +++- pkg/asset/manifests/template.go | 3 + pkg/asset/targets/targets.go | 2 + ...onfig-configmap-etcd-metrics-serving-ca.go | 64 ++++++++ ...shift-config-secret-etcd-metrics-client.go | 64 ++++++++ pkg/asset/tls/etcdmetrics.go | 146 ++++++++++++++++++ 7 files changed, 310 insertions(+), 8 deletions(-) create mode 100644 pkg/asset/templates/content/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.go create mode 100644 pkg/asset/templates/content/bootkube/openshift-config-secret-etcd-metrics-client.go create mode 100644 pkg/asset/tls/etcdmetrics.go diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index f6065ed1ed6..6fba0ce1e2e 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -79,6 +79,9 @@ func (a *Bootstrap) Dependencies() []asset.Asset { &tls.EtcdCA{}, &tls.EtcdCABundle{}, &tls.EtcdClientCertKey{}, + &tls.EtcdMetricsCABundle{}, + &tls.EtcdMetricsSignerClientCertKey{}, + &tls.EtcdMetricsSignerServerCertKey{}, &tls.EtcdSignerCertKey{}, &tls.EtcdSignerClientCertKey{}, &tls.JournalCertKey{}, @@ -388,6 +391,9 @@ func (a *Bootstrap) addParentFiles(dependencies asset.Parents) { &tls.EtcdCA{}, &tls.EtcdCABundle{}, &tls.EtcdClientCertKey{}, + &tls.EtcdMetricsCABundle{}, + &tls.EtcdMetricsSignerClientCertKey{}, + &tls.EtcdMetricsSignerServerCertKey{}, &tls.EtcdSignerCertKey{}, &tls.EtcdSignerClientCertKey{}, &tls.KubeAPIServerLBCABundle{}, diff --git a/pkg/asset/manifests/operators.go b/pkg/asset/manifests/operators.go index 595d16c71c0..b13999eb2f0 100644 --- a/pkg/asset/manifests/operators.go +++ b/pkg/asset/manifests/operators.go @@ -61,6 +61,8 @@ func (m *Manifests) Dependencies() []asset.Asset { &tls.RootCA{}, &tls.EtcdCA{}, &tls.EtcdClientCertKey{}, + &tls.EtcdMetricsCABundle{}, + &tls.EtcdMetricsSignerClientCertKey{}, &tls.MCSCertKey{}, &bootkube.KubeCloudConfig{}, @@ -71,6 +73,8 @@ func (m *Manifests) Dependencies() []asset.Asset { &bootkube.KubeSystemConfigmapEtcdServingCA{}, &bootkube.KubeSystemConfigmapRootCA{}, &bootkube.KubeSystemSecretEtcdClient{}, + &bootkube.OpenshiftConfigSecretEtcdMetricsClient{}, + &bootkube.OpenshiftConfigConfigmapEtcdMetricsServingCA{}, &bootkube.OpenshiftMachineConfigOperator{}, &bootkube.EtcdServiceKubeSystem{}, @@ -125,12 +129,16 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass etcdCA := &tls.EtcdCA{} mcsCertKey := &tls.MCSCertKey{} etcdClientCertKey := &tls.EtcdClientCertKey{} + etcdMetricsCABundle := &tls.EtcdMetricsCABundle{} + etcdMetricsSignerClientCertKey := &tls.EtcdMetricsSignerClientCertKey{} rootCA := &tls.RootCA{} dependencies.Get( clusterID, installConfig, etcdCA, etcdClientCertKey, + etcdMetricsCABundle, + etcdMetricsSignerClientCertKey, mcsCertKey, rootCA, ) @@ -145,6 +153,9 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass EtcdCaCert: string(etcdCA.Cert()), EtcdClientCert: base64.StdEncoding.EncodeToString(etcdClientCertKey.Cert()), EtcdClientKey: base64.StdEncoding.EncodeToString(etcdClientCertKey.Key()), + EtcdMetricsCaCert: string(etcdMetricsCABundle.Cert()), + EtcdMetricsClientCert: base64.StdEncoding.EncodeToString(etcdMetricsSignerClientCertKey.Cert()), + EtcdMetricsClientKey: base64.StdEncoding.EncodeToString(etcdMetricsSignerClientCertKey.Key()), McsTLSCert: base64.StdEncoding.EncodeToString(mcsCertKey.Cert()), McsTLSKey: base64.StdEncoding.EncodeToString(mcsCertKey.Key()), PullSecretBase64: base64.StdEncoding.EncodeToString([]byte(installConfig.Config.PullSecret)), @@ -162,6 +173,8 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass kubeSystemConfigmapEtcdServingCA := &bootkube.KubeSystemConfigmapEtcdServingCA{} kubeSystemConfigmapRootCA := &bootkube.KubeSystemConfigmapRootCA{} kubeSystemSecretEtcdClient := &bootkube.KubeSystemSecretEtcdClient{} + openshiftConfigSecretEtcdMetricsClient := &bootkube.OpenshiftConfigSecretEtcdMetricsClient{} + openshiftConfigConfigmapEtcdMetricsServingCA := &bootkube.OpenshiftConfigConfigmapEtcdMetricsServingCA{} openshiftMachineConfigOperator := &bootkube.OpenshiftMachineConfigOperator{} etcdServiceKubeSystem := &bootkube.EtcdServiceKubeSystem{} @@ -175,19 +188,23 @@ func (m *Manifests) generateBootKubeManifests(dependencies asset.Parents) []*ass kubeSystemConfigmapEtcdServingCA, kubeSystemConfigmapRootCA, kubeSystemSecretEtcdClient, + openshiftConfigSecretEtcdMetricsClient, + openshiftConfigConfigmapEtcdMetricsServingCA, openshiftMachineConfigOperator, etcdServiceKubeSystem, hostEtcdServiceKubeSystem, ) assetData := map[string][]byte{ - "kube-cloud-config.yaml": applyTemplateData(kubeCloudConfig.Files()[0].Data, templateData), - "machine-config-server-tls-secret.yaml": applyTemplateData(machineConfigServerTLSSecret.Files()[0].Data, templateData), - "pull.json": applyTemplateData(pull.Files()[0].Data, templateData), - "cvo-overrides.yaml": applyTemplateData(cVOOverrides.Files()[0].Data, templateData), - "host-etcd-service-endpoints.yaml": applyTemplateData(hostEtcdServiceEndpointsKubeSystem.Files()[0].Data, templateData), - "kube-system-configmap-etcd-serving-ca.yaml": applyTemplateData(kubeSystemConfigmapEtcdServingCA.Files()[0].Data, templateData), - "kube-system-configmap-root-ca.yaml": applyTemplateData(kubeSystemConfigmapRootCA.Files()[0].Data, templateData), - "kube-system-secret-etcd-client.yaml": applyTemplateData(kubeSystemSecretEtcdClient.Files()[0].Data, templateData), + "kube-cloud-config.yaml": applyTemplateData(kubeCloudConfig.Files()[0].Data, templateData), + "machine-config-server-tls-secret.yaml": applyTemplateData(machineConfigServerTLSSecret.Files()[0].Data, templateData), + "pull.json": applyTemplateData(pull.Files()[0].Data, templateData), + "cvo-overrides.yaml": applyTemplateData(cVOOverrides.Files()[0].Data, templateData), + "host-etcd-service-endpoints.yaml": applyTemplateData(hostEtcdServiceEndpointsKubeSystem.Files()[0].Data, templateData), + "kube-system-configmap-etcd-serving-ca.yaml": applyTemplateData(kubeSystemConfigmapEtcdServingCA.Files()[0].Data, templateData), + "kube-system-configmap-root-ca.yaml": applyTemplateData(kubeSystemConfigmapRootCA.Files()[0].Data, templateData), + "kube-system-secret-etcd-client.yaml": applyTemplateData(kubeSystemSecretEtcdClient.Files()[0].Data, templateData), + "openshift-config-secret-etcd-metrics-client.yaml": applyTemplateData(openshiftConfigSecretEtcdMetricsClient.Files()[0].Data, templateData), + "openshift-config-configmap-etcd-metrics-serving-ca.yaml": applyTemplateData(openshiftConfigConfigmapEtcdMetricsServingCA.Files()[0].Data, templateData), "04-openshift-machine-config-operator.yaml": []byte(openshiftMachineConfigOperator.Files()[0].Data), "etcd-service.yaml": []byte(etcdServiceKubeSystem.Files()[0].Data), diff --git a/pkg/asset/manifests/template.go b/pkg/asset/manifests/template.go index c2cf0769dbe..bbb4af2de20 100644 --- a/pkg/asset/manifests/template.go +++ b/pkg/asset/manifests/template.go @@ -21,6 +21,9 @@ type bootkubeTemplateData struct { EtcdCaCert string EtcdClientCert string EtcdClientKey string + EtcdMetricsCaCert string + EtcdMetricsClientCert string + EtcdMetricsClientKey string McsTLSCert string McsTLSKey string PullSecretBase64 string diff --git a/pkg/asset/targets/targets.go b/pkg/asset/targets/targets.go index bb153c322b4..e49ad54f68f 100644 --- a/pkg/asset/targets/targets.go +++ b/pkg/asset/targets/targets.go @@ -40,6 +40,8 @@ var ( &bootkube.OpenshiftMachineConfigOperator{}, &bootkube.EtcdServiceKubeSystem{}, &bootkube.HostEtcdServiceKubeSystem{}, + &bootkube.OpenshiftConfigSecretEtcdMetricsClient{}, + &bootkube.OpenshiftConfigConfigmapEtcdMetricsServingCA{}, &openshift.BindingDiscovery{}, &openshift.CloudCredsSecret{}, &openshift.KubeadminPasswordSecret{}, diff --git a/pkg/asset/templates/content/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.go b/pkg/asset/templates/content/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.go new file mode 100644 index 00000000000..45281fcf1ed --- /dev/null +++ b/pkg/asset/templates/content/bootkube/openshift-config-configmap-etcd-metrics-serving-ca.go @@ -0,0 +1,64 @@ +package bootkube + +import ( + "os" + "path/filepath" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/templates/content" +) + +const ( + openshiftConfigConfigmapEtcdMetricsServingCAFileName = "openshift-config-configmap-etcd-metrics-serving-ca.yaml.template" +) + +var _ asset.WritableAsset = (*OpenshiftConfigConfigmapEtcdMetricsServingCA)(nil) + +// OpenshiftConfigConfigmapEtcdMetricsServingCA is the constant to represent contents of openshift-config-configmap-etcd-metrics-serving-ca.yaml.template file. +type OpenshiftConfigConfigmapEtcdMetricsServingCA struct { + FileList []*asset.File +} + +// Dependencies returns all of the dependencies directly needed by the asset +func (t *OpenshiftConfigConfigmapEtcdMetricsServingCA) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Name returns the human-friendly name of the asset. +func (t *OpenshiftConfigConfigmapEtcdMetricsServingCA) Name() string { + return "OpenshiftConfigConfigmapEtcdMetricsServingCA" +} + +// Generate generates the actual files by this asset +func (t *OpenshiftConfigConfigmapEtcdMetricsServingCA) Generate(parents asset.Parents) error { + fileName := openshiftConfigConfigmapEtcdMetricsServingCAFileName + data, err := content.GetBootkubeTemplate(fileName) + if err != nil { + return err + } + t.FileList = []*asset.File{ + { + Filename: filepath.Join(content.TemplateDir, fileName), + Data: []byte(data), + }, + } + return nil +} + +// Files returns the files generated by the asset. +func (t *OpenshiftConfigConfigmapEtcdMetricsServingCA) Files() []*asset.File { + return t.FileList +} + +// Load returns the asset from disk. +func (t *OpenshiftConfigConfigmapEtcdMetricsServingCA) Load(f asset.FileFetcher) (bool, error) { + file, err := f.FetchByName(filepath.Join(content.TemplateDir, openshiftConfigConfigmapEtcdMetricsServingCAFileName)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + t.FileList = []*asset.File{file} + return true, nil +} diff --git a/pkg/asset/templates/content/bootkube/openshift-config-secret-etcd-metrics-client.go b/pkg/asset/templates/content/bootkube/openshift-config-secret-etcd-metrics-client.go new file mode 100644 index 00000000000..802787e9ad9 --- /dev/null +++ b/pkg/asset/templates/content/bootkube/openshift-config-secret-etcd-metrics-client.go @@ -0,0 +1,64 @@ +package bootkube + +import ( + "os" + "path/filepath" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/templates/content" +) + +const ( + openshiftConfigSecretEtcdMetricsClientFileName = "openshift-config-secret-etcd-metrics-client.yaml.template" +) + +var _ asset.WritableAsset = (*OpenshiftConfigSecretEtcdMetricsClient)(nil) + +// OpenshiftConfigSecretEtcdMetricsClient is the constant to represent contents of openshift-config-secret-etcd-metrics-client.yaml.template file. +type OpenshiftConfigSecretEtcdMetricsClient struct { + FileList []*asset.File +} + +// Dependencies returns all of the dependencies directly needed by the asset +func (t *OpenshiftConfigSecretEtcdMetricsClient) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Name returns the human-friendly name of the asset. +func (t *OpenshiftConfigSecretEtcdMetricsClient) Name() string { + return "OpenshiftConfigSecretEtcdMetricsClient" +} + +// Generate generates the actual files by this asset +func (t *OpenshiftConfigSecretEtcdMetricsClient) Generate(parents asset.Parents) error { + fileName := openshiftConfigSecretEtcdMetricsClientFileName + data, err := content.GetBootkubeTemplate(fileName) + if err != nil { + return err + } + t.FileList = []*asset.File{ + { + Filename: filepath.Join(content.TemplateDir, fileName), + Data: []byte(data), + }, + } + return nil +} + +// Files returns the files generated by the asset. +func (t *OpenshiftConfigSecretEtcdMetricsClient) Files() []*asset.File { + return t.FileList +} + +// Load returns the asset from disk. +func (t *OpenshiftConfigSecretEtcdMetricsClient) Load(f asset.FileFetcher) (bool, error) { + file, err := f.FetchByName(filepath.Join(content.TemplateDir, openshiftConfigSecretEtcdMetricsClientFileName)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + t.FileList = []*asset.File{file} + return true, nil +} diff --git a/pkg/asset/tls/etcdmetrics.go b/pkg/asset/tls/etcdmetrics.go new file mode 100644 index 00000000000..f377989b691 --- /dev/null +++ b/pkg/asset/tls/etcdmetrics.go @@ -0,0 +1,146 @@ +package tls + +import ( + "crypto/x509" + "crypto/x509/pkix" + + "github.com/openshift/installer/pkg/asset" +) + +// EtcdMetricsSignerCertKey is a key/cert pair that signs the etcd-metrics client and peer certs. +type EtcdMetricsSignerCertKey struct { + SelfSignedCertKey +} + +var _ asset.WritableAsset = (*EtcdMetricsSignerCertKey)(nil) + +// Dependencies returns the dependency of the root-ca, which is empty. +func (c *EtcdMetricsSignerCertKey) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Generate generates the root-ca key and cert pair. +func (c *EtcdMetricsSignerCertKey) Generate(parents asset.Parents) error { + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "etcd-metrics-signer", OrganizationalUnit: []string{"openshift"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + Validity: ValidityTenYears, + IsCA: true, + } + + return c.SelfSignedCertKey.Generate(cfg, "etcd-metrics-signer") +} + +// Name returns the human-friendly name of the asset. +func (c *EtcdMetricsSignerCertKey) Name() string { + return "Certificate (etcd-metrics-signer)" +} + +// EtcdMetricsCABundle is the asset the generates the etcd-metrics-ca-bundle, +// which contains all the individual client CAs. +type EtcdMetricsCABundle struct { + CertBundle +} + +var _ asset.Asset = (*EtcdMetricsCABundle)(nil) + +// Dependencies returns the dependency of the cert bundle. +func (a *EtcdMetricsCABundle) Dependencies() []asset.Asset { + return []asset.Asset{ + &EtcdMetricsSignerCertKey{}, + } +} + +// Generate generates the cert bundle based on its dependencies. +func (a *EtcdMetricsCABundle) Generate(deps asset.Parents) error { + var certs []CertInterface + for _, asset := range a.Dependencies() { + deps.Get(asset) + certs = append(certs, asset.(CertInterface)) + } + return a.CertBundle.Generate("etcd-metrics-ca-bundle", certs...) +} + +// Name returns the human-friendly name of the asset. +func (a *EtcdMetricsCABundle) Name() string { + return "Certificate (etcd-metrics-ca-bundle)" +} + +// EtcdMetricsSignerClientCertKey is the asset that generates the etcd-metrics client key/cert pair. +type EtcdMetricsSignerClientCertKey struct { + SignedCertKey +} + +var _ asset.Asset = (*EtcdMetricsSignerClientCertKey)(nil) + +// Dependencies returns the dependency of the the cert/key pair, which includes +// the parent CA, and install config if it depends on the install config for +// DNS names, etc. +func (a *EtcdMetricsSignerClientCertKey) Dependencies() []asset.Asset { + return []asset.Asset{ + &EtcdMetricsSignerCertKey{}, + } +} + +// Generate generates the cert/key pair based on its dependencies. +func (a *EtcdMetricsSignerClientCertKey) Generate(dependencies asset.Parents) error { + ca := &EtcdMetricsSignerCertKey{} + dependencies.Get(ca) + + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "etcd-metrics", OrganizationalUnit: []string{"etcd-metrics"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + Validity: ValidityTenYears, + } + + return a.SignedCertKey.Generate(cfg, ca, "etcd-metrics-signer-client", DoNotAppendParent) +} + +// Name returns the human-friendly name of the asset. +func (a *EtcdMetricsSignerClientCertKey) Name() string { + return "Certificate (etcd-metrics-signer-client)" +} + +// EtcdMetricsSignerServerCertKey is the asset that generates the etcd-metrics server key/cert pair. +type EtcdMetricsSignerServerCertKey struct { + SignedCertKey +} + +var _ asset.Asset = (*EtcdMetricsSignerServerCertKey)(nil) + +// Dependencies returns the dependency of the the cert/key pair, which includes +// the parent CA, and install config if it depends on the install config for +// DNS names, etc. +func (a *EtcdMetricsSignerServerCertKey) Dependencies() []asset.Asset { + return []asset.Asset{ + &EtcdMetricsSignerCertKey{}, + } +} + +// Generate generates the cert/key pair based on its dependencies. +func (a *EtcdMetricsSignerServerCertKey) Generate(dependencies asset.Parents) error { + ca := &EtcdMetricsSignerCertKey{} + dependencies.Get(ca) + + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "etcd-metrics", OrganizationalUnit: []string{"etcd-metrics"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{ + "etcd", + "etcd.kube-system", + "etcd.kube-system.svc.cluster.local", + "etcd.kube-system.svc", + "localhost", + }, + Validity: ValidityTenYears, + } + + return a.SignedCertKey.Generate(cfg, ca, "etcd-metrics-signer-server", DoNotAppendParent) +} + +// Name returns the human-friendly name of the asset. +func (a *EtcdMetricsSignerServerCertKey) Name() string { + return "Certificate (etcd-metrics-signer-server)" +}