Create JKS for internode encryption (#127) (#156)

* Refactor out webhook and cryptography code (#127)

* Intranode crypto intermediate commit

* Intranode crypto intermediate commit

* Fix return value change

* Replace intra with inter to fix wording

* Encryption test yaml file

* Use correct private key in jks generation

* Tidy go modules

* Use good test name namespace and description

* Remove unintentional file add

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1
 	github.com/operator-framework/operator-sdk v0.17.0
+	github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0

go.sum

@@ -763,6 +763,8 @@ github.com/otiai10/mint v1.2.4/go.mod h1:d+b7n/0R3tdyUYYylALXpWQ/kTN+QobSq/4SRGB
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible h1:Jd6xfriVlJ6hWPvYOE0Ni0QWcNTLRehfGPFxr3eSL80=
+github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

operator/cmd/manager/main.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -19,6 +18,7 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
 
+	webhook "github.com/datastax/cass-operator/operator/pkg/admissionwebhook"
 	"github.com/datastax/cass-operator/operator/pkg/apis"
 	"github.com/datastax/cass-operator/operator/pkg/controller"
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
@@ -47,13 +47,6 @@ var (
 	metricsPort         int32 = 8383
 	operatorMetricsPort int32 = 8686
 	version                   = "DEV"
-
-	altCertDir = filepath.Join(os.TempDir()) //Alt directory is necessary because regular key/cert mountpoint is read-only
-	certDir    = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
-
-	serverCertFile    = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs", "tls.crt")
-	altServerCertFile = filepath.Join(altCertDir, "tls.crt")
-	altServerKeyFile  = filepath.Join(altCertDir, "tls.key")
 )
 var log = logf.Log.WithName("cmd")
 
@@ -127,10 +120,11 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err = ensureWebhookConfigVolume(cfg); err != nil {
+	if err = webhook.EnsureWebhookConfigVolume(cfg); err != nil {
 		log.Error(err, "Failed to ensure webhook volume")
 	}
-	if err = ensureWebhookCertificate(cfg); err != nil {
+	var certDir string
+	if certDir, err = webhook.EnsureWebhookCertificate(cfg); err != nil {
 		log.Error(err, "Failed to ensure webhook CA configuration")
 	}
 

operator/cmd/manager/webhook.go → operator/pkg/admissionwebhook/webhook.go

@@ -1,4 +1,4 @@
-package main
+package webhook
 
 import (
 	"context"
@@ -9,26 +9,40 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
+	"github.com/datastax/cass-operator/operator/pkg/utils"
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func ensureWebhookCertificate(cfg *rest.Config) (err error) {
+var (
+	altCertDir = filepath.Join(os.TempDir()) //Alt directory is necessary because regular key/cert mountpoint is read-only
+	certDir    = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
+
+	serverCertFile    = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs", "tls.crt")
+	altServerCertFile = filepath.Join(altCertDir, "tls.crt")
+	altServerKeyFile  = filepath.Join(altCertDir, "tls.key")
+
+	log = logf.Log.WithName("cmd")
+)
+
+func EnsureWebhookCertificate(cfg *rest.Config) (certDir string, err error) {
 	var contents []byte
 	var webhook map[string]interface{}
 	var bundled string
 	var client crclient.Client
 	namespace, err := k8sutil.GetOperatorNamespace()
 	if err != nil {
-		return err
+		return "", err
 	}
 	var certpool *x509.CertPool
 	if contents, err = ioutil.ReadFile(serverCertFile); err == nil && len(contents) > 0 {
@@ -52,7 +66,7 @@ func ensureWebhookCertificate(cfg *rest.Config) (err error) {
 								}
 								if _, err = cert.Verify(verify_opts); err == nil {
 									log.Info("Found valid certificate for webhook")
-									return nil
+									return certDir, nil
 								}
 							}
 						}
@@ -64,10 +78,10 @@ func ensureWebhookCertificate(cfg *rest.Config) (err error) {
 	return updateSecretAndWebhook(cfg, namespace)
 }
 
-func updateSecretAndWebhook(cfg *rest.Config, namespace string) (err error) {
+func updateSecretAndWebhook(cfg *rest.Config, namespace string) (certDir string, err error) {
 	var key, cert string
 	var client crclient.Client
-	if key, cert, err = getNewCertAndKey(namespace); err == nil {
+	if key, cert, err = utils.GetNewCAandKey("cass-operator-webhook-config", namespace); err == nil {
 		if client, err = crclient.New(cfg, crclient.Options{}); err == nil {
 			secret := &v1.Secret{}
 			err = client.Get(context.Background(), crclient.ObjectKey{
@@ -84,7 +98,7 @@ func updateSecretAndWebhook(cfg *rest.Config, namespace string) (err error) {
 						if err = ioutil.WriteFile(altServerKeyFile, []byte(key), 0600); err == nil {
 							certDir = altCertDir
 							log.Info("TLS secret updated in pod mount")
-							return updateWebhook(client, cert, namespace)
+							return certDir, updateWebhook(client, cert, namespace)
 						}
 					}
 				}
@@ -93,7 +107,7 @@ func updateSecretAndWebhook(cfg *rest.Config, namespace string) (err error) {
 		}
 	}
 	log.Error(err, "Failed to update certificates")
-	return err
+	return certDir, err
 }
 
 func fetchWebhookForNamespace(client crclient.Client, namespace string) (err error, webhook_config *unstructured.Unstructured, webhook map[string]interface{}, unstructured_index int) {
@@ -151,7 +165,7 @@ func updateWebhook(client crclient.Client, cert, namespace string) (err error) {
 	return err
 }
 
-func ensureWebhookConfigVolume(cfg *rest.Config) (err error) {
+func EnsureWebhookConfigVolume(cfg *rest.Config) (err error) {
 	var pod *v1.Pod
 	namespace, err := k8sutil.GetOperatorNamespace()
 	if err != nil {

operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -663,11 +663,11 @@ func (dc *CassandraDatacenter) GetContainerPorts() ([]corev1.ContainerPort, erro
 			ContainerPort: 8609,
 		},
 		{
-			Name:          "intra-node",
+			Name:          "inter-node",
 			ContainerPort: int32(broadcastPort),
 		},
 		{
-			Name:          "tls-intra-node",
+			Name:          "tls-inter-node",
 			ContainerPort: 7001,
 		},
 		// jmx-port 7199 was here, seems like we no longer need to expose it

operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types_test.go

@@ -241,10 +241,10 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) {
 					Name:          "inter-node-msg",
 					ContainerPort: 8609,
 				}, {
-					Name:          "intra-node",
+					Name:          "inter-node",
 					ContainerPort: DefaultBroadcastPort,
 				}, {
-					Name:          "tls-intra-node",
+					Name:          "tls-inter-node",
 					ContainerPort: 7001,
 				}, {
 					Name:          "mgmt-api-http",
@@ -268,10 +268,10 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) {
 					Name:          "inter-node-msg",
 					ContainerPort: 8609,
 				}, {
-					Name:          "intra-node",
+					Name:          "inter-node",
 					ContainerPort: DefaultBroadcastPort,
 				}, {
-					Name:          "tls-intra-node",
+					Name:          "tls-inter-node",
 					ContainerPort: 7001,
 				}, {
 					Name:          "mgmt-api-http",
@@ -297,10 +297,10 @@ func TestCassandraDatacenter_GetContainerPorts(t *testing.T) {
 					Name:          "inter-node-msg",
 					ContainerPort: 8609,
 				}, {
-					Name:          "intra-node",
+					Name:          "inter-node",
 					ContainerPort: DefaultBroadcastPort,
 				}, {
-					Name:          "tls-intra-node",
+					Name:          "tls-inter-node",
 					ContainerPort: 7001,
 				}, {
 					Name:          "mgmt-api-http",

operator/pkg/reconciliation/constructor.go

@@ -508,6 +508,10 @@ func buildContainers(dc *api.CassandraDatacenter, serverVolumeMounts []corev1.Vo
 		Name:      pvcName,
 		MountPath: "/var/lib/cassandra",
 	})
+	serverVolumeMounts = append(serverVolumeMounts, corev1.VolumeMount{
+		Name:      "encryption-cred-storage",
+		MountPath: "/etc/encryption/",
+	})
 	cassContainer.VolumeMounts = serverVolumeMounts
 
 	// server logger container
@@ -590,21 +594,7 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, zone string, rackName str
 	affinity.PodAntiAffinity = calculatePodAntiAffinity(dc.Spec.AllowMultipleNodesPerWorker)
 	baseTemplate.Spec.Affinity = affinity
 
-	// volumes
-	vServerConfig := corev1.Volume{}
-	vServerConfig.Name = "server-config"
-	vServerConfig.VolumeSource = corev1.VolumeSource{
-		EmptyDir: &corev1.EmptyDirVolumeSource{},
-	}
-
-	vServerLogs := corev1.Volume{}
-	vServerLogs.Name = "server-logs"
-	vServerLogs.VolumeSource = corev1.VolumeSource{
-		EmptyDir: &corev1.EmptyDirVolumeSource{},
-	}
-
-	volumes := []corev1.Volume{vServerConfig, vServerLogs}
-	baseTemplate.Spec.Volumes = append(baseTemplate.Spec.Volumes, volumes...)
+	addVolumes(dc, baseTemplate)
 
 	serviceAccount := "default"
 	if dc.Spec.ServiceAccount != "" {
@@ -676,3 +666,30 @@ func buildInitReaperSchemaJob(dc *api.CassandraDatacenter) *v1.Job {
 		},
 	}
 }
+
+func addVolumes(dc *api.CassandraDatacenter,baseTemplate *corev1.PodTemplateSpec) []corev1.Volume {
+	vServerConfig := corev1.Volume{}
+	vServerConfig.Name = "server-config"
+	vServerConfig.VolumeSource = corev1.VolumeSource{
+		EmptyDir: &corev1.EmptyDirVolumeSource{},
+	}
+
+	vServerLogs := corev1.Volume{}
+	vServerLogs.Name = "server-logs"
+	vServerLogs.VolumeSource = corev1.VolumeSource{
+		EmptyDir: &corev1.EmptyDirVolumeSource{},
+	}
+
+	vServerEncryption := corev1.Volume{}
+	vServerEncryption.Name = "encryption-cred-storage"
+	vServerEncryption.VolumeSource = corev1.VolumeSource{
+		Secret: &corev1.SecretVolumeSource{
+			SecretName:fmt.Sprintf("%s-keystore", dc.Name)},
+	}
+
+
+	volumes := []corev1.Volume{vServerConfig, vServerLogs, vServerEncryption}
+	baseTemplate.Spec.Volumes = append(baseTemplate.Spec.Volumes, volumes...)
+	return volumes
+
+}