Store transport keys and certificates in a single shared secret.

This facilitates a move to StatefulSets where the mounted secrets must be the same between
all the Pods in the same StatefulSet

operators/pkg/controller/elasticsearch/certificates/ca_reconcile.go

@@ -107,7 +107,7 @@ func Reconcile(
 	}
 
 	// reconcile transport certificates
-	result, err := transport.ReconcileTransportCertificateSecrets(
+	result, err := transport.ReconcileTransportCertificatesSecrets(
 		c,
 		scheme,
 		transportCA,

operators/pkg/controller/elasticsearch/certificates/transport/csr.go

@@ -18,8 +18,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-// CreateValidatedCertificateTemplate validates a CSR and creates a certificate template.
-func CreateValidatedCertificateTemplate(
+// createValidatedCertificateTemplate validates a CSR and creates a certificate template.
+func createValidatedCertificateTemplate(
 	pod corev1.Pod,
 	cluster v1alpha1.Elasticsearch,
 	svcs []corev1.Service,

operators/pkg/controller/elasticsearch/certificates/transport/csr_test.go

@@ -35,8 +35,8 @@ func Test_createValidatedCertificateTemplate(t *testing.T) {
 	// we expect this name to be used for both the common name as well as the es othername
 	cn := "test-pod-name.node.test-es-name.test-namespace.es.local"
 
-	validatedCert, err := CreateValidatedCertificateTemplate(
-		testPod, testCluster, []corev1.Service{testSvc}, testCSR, certificates.DefaultCertValidity,
+	validatedCert, err := createValidatedCertificateTemplate(
+		testPod, testES, []corev1.Service{testSvc}, testCSR, certificates.DefaultCertValidity,
 	)
 	require.NoError(t, err)
 
@@ -86,7 +86,7 @@ func Test_buildGeneralNames(t *testing.T) {
 		{
 			name: "no svcs and user-provided SANs",
 			args: args{
-				cluster: testCluster,
+				cluster: testES,
 				pod:     testPod,
 			},
 			want: []certificates.GeneralName{

operators/pkg/controller/elasticsearch/certificates/transport/pod_secret.go

@@ -5,80 +5,216 @@
 package transport
 
 import (
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"fmt"
+	"reflect"
+	"time"
+
 	"github.com/elastic/cloud-on-k8s/operators/pkg/apis/elasticsearch/v1alpha1"
-	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/reconciler"
-	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/label"
-	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/name"
-	"github.com/elastic/cloud-on-k8s/operators/pkg/utils/k8s"
+	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/certificates"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 )
 
-const (
-	// LabelCertificateType is a label key that specifies what type of certificates the secret contains
-	LabelCertificateType = "certificates.elasticsearch.k8s.elastic.co/type"
-	// LabelCertificateTypeTransport is the LabelCertificateType value used for transport certificates
-	LabelCertificateTypeTransport = "transport"
-)
+// PodKeyFileName returns the name of the private key entry for a specific pod in a transport certificates secret.
+func PodKeyFileName(podName string) string {
+	return fmt.Sprintf("%s.%s", podName, certificates.KeyFileName)
+}
 
-// EnsureTransportCertificateSecretExists ensures the existence and Labels of the corev1.Secret that at a later point
-// in time will contain the transport certificates.
-func EnsureTransportCertificateSecretExists(
-	c k8s.Client,
-	scheme *runtime.Scheme,
+// PodCertFileName returns the name of the certificates entry for a specific pod in a transport certificates secret.
+func PodCertFileName(podName string) string {
+	return fmt.Sprintf("%s.%s", podName, certificates.CertFileName)
+}
+
+// ensureTransportCertificatesSecretContentsForPod ensures that the transport certificates secret has the correct
+// content for a specific pod
+func ensureTransportCertificatesSecretContentsForPod(
 	es v1alpha1.Elasticsearch,
+	secret *corev1.Secret,
 	pod corev1.Pod,
-) (*corev1.Secret, error) {
-	expected := corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: pod.Namespace,
-			Name:      name.TransportCertsSecret(pod.Name),
-
-			Labels: map[string]string{
-				// a label that allows us to list secrets of this type
-				LabelCertificateType: LabelCertificateTypeTransport,
-				// a label referencing the related pod so we can look up the pod from this secret
-				label.PodNameLabelName: pod.Name,
-				// a label showing which cluster this pod belongs to
-				label.ClusterNameLabelName: es.Name,
-			},
-		},
+	svcs []corev1.Service,
+	ca *certificates.CA,
+	rotationParams certificates.RotationParams,
+) error {
+	// verify that the secret contains a parsable private key, create if it does not exist
+	var privateKey *rsa.PrivateKey
+	needsNewPrivateKey := true
+	if privateKeyData, ok := secret.Data[PodKeyFileName(pod.Name)]; ok {
+		storedPrivateKey, err := certificates.ParsePEMPrivateKey(privateKeyData)
+		if err != nil {
+			log.Error(err, "Unable to parse stored private key", "pod", pod.Name)
+		} else {
+			needsNewPrivateKey = false
+			privateKey = storedPrivateKey
+		}
+	}
+
+	// if we need a new private key, generate it
+	if needsNewPrivateKey {
+		generatedPrivateKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
+		if err != nil {
+			return err
+		}
+
+		privateKey = generatedPrivateKey
+		secret.Data[PodKeyFileName(pod.Name)] = certificates.EncodePEMPrivateKey(*privateKey)
+	}
+
+	if shouldIssueNewCertificate(es, *secret, pod, privateKey, svcs, ca, rotationParams.RotateBefore) {
+		log.Info(
+			"Issuing new certificate",
+			"pod", pod.Name,
+		)
+
+		csr, err := x509.CreateCertificateRequest(cryptorand.Reader, &x509.CertificateRequest{}, privateKey)
+		if err != nil {
+			return err
+		}
+
+		// create a cert from the csr
+		parsedCSR, err := x509.ParseCertificateRequest(csr)
+		if err != nil {
+			return err
+		}
+
+		validatedCertificateTemplate, err := createValidatedCertificateTemplate(
+			pod, es, svcs, parsedCSR, rotationParams.Validity,
+		)
+		if err != nil {
+			return err
+		}
+		// sign the certificate
+		certData, err := ca.CreateCertificate(*validatedCertificateTemplate)
+		if err != nil {
+			return err
+		}
+
+		// store the issued certificate in a secret mounted into the pod
+		secret.Data[PodCertFileName(pod.Name)] = certificates.EncodePEMCert(certData, ca.Cert.Raw)
+	}
+
+	return nil
+}
+
+// shouldIssueNewCertificate returns true if we should issue a new certificate.
+//
+// Reasons for reissuing a certificate:
+// - no certificate yet
+// - certificate has the wrong format
+// - certificate is invalid or expired
+// - certificate SAN and IP does not match pod SAN and IP
+func shouldIssueNewCertificate(
+	es v1alpha1.Elasticsearch,
+	secret corev1.Secret,
+	pod corev1.Pod,
+	privateKey *rsa.PrivateKey,
+	svcs []corev1.Service,
+	ca *certificates.CA,
+	certReconcileBefore time.Duration,
+) bool {
+	certCommonName := buildCertificateCommonName(pod, es.Name, es.Namespace)
+
+	generalNames, err := buildGeneralNames(es, svcs, pod)
+	if err != nil {
+		log.Error(err, "Cannot create GeneralNames for the TLS certificate", "pod", pod.Name)
+		return true
 	}
 
-	// reconcile the secret resource
-	var reconciled corev1.Secret
-	if err := reconciler.ReconcileResource(reconciler.Params{
-		Client:     c,
-		Scheme:     scheme,
-		Owner:      &es,
-		Expected:   &expected,
-		Reconciled: &reconciled,
-		NeedsUpdate: func() bool {
-			// we only care about labels, not contents at this point, and we can allow additional labels
-			if reconciled.Labels == nil {
-				return true
-			}
-
-			for k, v := range expected.Labels {
-				if rv, ok := reconciled.Labels[k]; !ok || rv != v {
-					return true
-				}
-			}
-			return false
-		},
-		UpdateReconciled: func() {
-			if reconciled.Labels == nil {
-				reconciled.Labels = expected.Labels
-			} else {
-				for k, v := range expected.Labels {
-					reconciled.Labels[k] = v
-				}
-			}
-		},
-	}); err != nil {
-		return nil, err
+	cert := extractTransportCert(secret, pod, certCommonName)
+	if cert == nil {
+		return true
 	}
 
-	return &reconciled, nil
+	publicKey, publicKeyOk := cert.PublicKey.(*rsa.PublicKey)
+	if !publicKeyOk || publicKey.N.Cmp(privateKey.PublicKey.N) != 0 || publicKey.E != privateKey.PublicKey.E {
+		log.Info(
+			"Certificate belongs do a different public key, should issue new",
+			"subject", cert.Subject,
+			"issuer", cert.Issuer,
+			"current_ca_subject", ca.Cert.Subject,
+			"pod", pod.Name,
+		)
+		return true
+	}
+
+	pool := x509.NewCertPool()
+	pool.AddCert(ca.Cert)
+	verifyOpts := x509.VerifyOptions{
+		DNSName:       certCommonName,
+		Roots:         pool,
+		Intermediates: pool,
+	}
+	if _, err := cert.Verify(verifyOpts); err != nil {
+		log.Info(
+			fmt.Sprintf("Certificate was not valid, should issue new: %s", err),
+			"subject", cert.Subject,
+			"issuer", cert.Issuer,
+			"current_ca_subject", ca.Cert.Subject,
+			"pod", pod.Name,
+		)
+		return true
+	}
+
+	if time.Now().After(cert.NotAfter.Add(-certReconcileBefore)) {
+		log.Info("Certificate soon to expire, should issue new", "pod", pod.Name)
+		return true
+	}
+
+	// compare actual vs. expected SANs
+	expected, err := certificates.MarshalToSubjectAlternativeNamesData(generalNames)
+	if err != nil {
+		log.Error(err, "Cannot marshal subject alternative names", "pod", pod.Name)
+		return true
+	}
+	extraExtensionFound := false
+	for _, ext := range cert.Extensions {
+		if !ext.Id.Equal(certificates.SubjectAlternativeNamesObjectIdentifier) {
+			continue
+		}
+		extraExtensionFound = true
+		if !reflect.DeepEqual(ext.Value, expected) {
+			log.Info("Certificate SANs do not match expected one, should issue new", "pod", pod.Name)
+			return true
+		}
+	}
+	if !extraExtensionFound {
+		log.Info("SAN extra extension not found, should issue new certificate", "pod", pod.Name)
+		return true
+	}
+
+	return false
+}
+
+// extractTransportCert extracts the transport certificate for the pod with the commonName from the Secret
+func extractTransportCert(secret corev1.Secret, pod corev1.Pod, commonName string) *x509.Certificate {
+	certData, ok := secret.Data[PodCertFileName(pod.Name)]
+	if !ok {
+		log.Info("No tls certificate found in secret", "pod", pod.Name)
+		return nil
+	}
+
+	certs, err := certificates.ParsePEMCerts(certData)
+	if err != nil {
+		log.Error(err, "Invalid certificate data found, issuing new certificate", "pod", pod.Name)
+		return nil
+	}
+
+	// look for the certificate based on the CommonName
+	var names []string
+	for _, c := range certs {
+		if c.Subject.CommonName == commonName {
+			return c
+		}
+		names = append(names, c.Subject.CommonName)
+	}
+
+	log.Info(
+		"Did not found a certificate with the expected common name",
+		"pod", pod.Name,
+		"expected", commonName,
+		"found", names,
+	)
+
+	return nil
 }