Persist encrypted credentials under the credentials/ directory

so that we can prevent unnecessary node replacement when `kube-aws update` run.

Encrypted credentials are named with the suffix `.enc` hence `*-key.pem.enc` for keys, `*.pem.env` for certs, and `ca.pem.enc` for ca cert.

If you've removed one of more `*.enc` files, `kube-aws (validate|up)` automatically re-generate not only removed ones but "all" the `*.enc` files by encrypting pem files.

The whole file tree representing kube-aws' state after `kube-aws init` now look like:

```
$ tree e2e/assets/
e2e/assets/
└── kubeawstest2
    ├── cluster.yaml
    ├── credentials
    │   ├── admin-key.pem
    │   ├── admin-key.pem.enc
    │   ├── admin.pem
    │   ├── admin.pem.enc
    │   ├── apiserver-key.pem
    │   ├── apiserver-key.pem.enc
    │   ├── apiserver.pem
    │   ├── apiserver.pem.enc
    │   ├── ca-key.pem
    │   ├── ca-key.pem.enc
    │   ├── ca.pem
    │   ├── ca.pem.enc
    │   ├── etcd-client-key.pem
    │   ├── etcd-client-key.pem.enc
    │   ├── etcd-client.pem
    │   ├── etcd-client.pem.enc
    │   ├── etcd-key.pem
    │   ├── etcd-key.pem.enc
    │   ├── etcd.pem
    │   ├── etcd.pem.enc
    │   ├── worker-key.pem
    │   ├── worker-key.pem.enc
    │   ├── worker.pem
    │   └── worker.pem.enc
    ├── kubeconfig
    ├── stack-template.json
    └── userdata
        ├── cloud-config-controller
        ├── cloud-config-etcd
        └── cloud-config-worker

3 directories, 30 files
```

fixes #107

config/config.go

@@ -12,9 +12,6 @@ import (
 	"strings"
 	"unicode/utf8"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/kms"
 	"github.com/coreos/go-semver/semver"
 	"github.com/coreos/kube-aws/coreos/amiregistry"
 	"github.com/coreos/kube-aws/coreos/userdatavalidation"
@@ -542,32 +539,18 @@ type stackConfig struct {
 }
 
 func (c Cluster) stackConfig(opts StackTemplateOptions, compressUserData bool) (*stackConfig, error) {
-	assets, err := ReadTLSAssets(opts.TLSAssetsDir)
-	if err != nil {
-		return nil, err
-	}
+	var err error
 	stackConfig := stackConfig{}
 
 	if stackConfig.Config, err = c.Config(); err != nil {
 		return nil, err
 	}
 
-	awsConfig := aws.NewConfig().
-		WithRegion(stackConfig.Config.Region).
-		WithCredentialsChainVerboseErrors(true)
-
-	// TODO Cleaner way to inject this dependency
-	var kmsSvc EncryptService
-	if c.providedEncryptService != nil {
-		kmsSvc = c.providedEncryptService
-	} else {
-		kmsSvc = kms.New(session.New(awsConfig))
-	}
-
-	compactAssets, err := assets.Compact(stackConfig.Config.KMSKeyARN, kmsSvc)
-	if err != nil {
-		return nil, fmt.Errorf("failed to compress TLS assets: %v", err)
-	}
+	compactAssets, err := ReadOrCreateCompactTLSAssets(opts.TLSAssetsDir, KMSConfig{
+		Region:         stackConfig.Config.Region,
+		KMSKeyARN:      c.KMSKeyARN,
+		EncryptService: c.providedEncryptService,
+	})
 
 	stackConfig.Config.TLSConfig = compactAssets
 

config/tls_config.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/kms"
 	"github.com/coreos/kube-aws/gzipcompressor"
 	"github.com/coreos/kube-aws/netutil"
@@ -32,7 +33,23 @@ type RawTLSAssets struct {
 	EtcdClientKey  []byte
 }
 
-// PEM -> gzip -> base64 encoded TLS assets.
+// Encrypted PEM encoded TLS assets
+type EncryptedTLSAssets struct {
+	CACert         []byte
+	CAKey          []byte
+	APIServerCert  []byte
+	APIServerKey   []byte
+	WorkerCert     []byte
+	WorkerKey      []byte
+	AdminCert      []byte
+	AdminKey       []byte
+	EtcdCert       []byte
+	EtcdClientCert []byte
+	EtcdKey        []byte
+	EtcdClientKey  []byte
+}
+
+// PEM -> encrypted -> gzip -> base64 encoded TLS assets.
 type CompactTLSAssets struct {
 	CACert         string
 	CAKey          string
@@ -174,7 +191,7 @@ func (c *Cluster) NewTLSAssets(caKey *rsa.PrivateKey, caCert *x509.Certificate)
 	}, nil
 }
 
-func ReadTLSAssets(dirname string) (*RawTLSAssets, error) {
+func ReadRawTLSAssets(dirname string) (*RawTLSAssets, error) {
 	r := new(RawTLSAssets)
 	files := []struct {
 		name      string
@@ -208,6 +225,40 @@ func ReadTLSAssets(dirname string) (*RawTLSAssets, error) {
 	return r, nil
 }
 
+func ReadEncryptedTLSAssets(dirname string) (*EncryptedTLSAssets, error) {
+	r := new(EncryptedTLSAssets)
+	files := []struct {
+		name      string
+		cert, key *[]byte
+	}{
+		{"ca", &r.CACert, nil},
+		{"apiserver", &r.APIServerCert, &r.APIServerKey},
+		{"worker", &r.WorkerCert, &r.WorkerKey},
+		{"admin", &r.AdminCert, &r.AdminKey},
+		{"etcd", &r.EtcdCert, &r.EtcdKey},
+		{"etcd-client", &r.EtcdClientCert, &r.EtcdClientKey},
+	}
+	for _, file := range files {
+		certPath := filepath.Join(dirname, file.name+".pem.enc")
+		keyPath := filepath.Join(dirname, file.name+"-key.pem.enc")
+
+		certData, err := ioutil.ReadFile(certPath)
+		if err != nil {
+			return nil, err
+		}
+		*file.cert = certData
+
+		if file.key != nil {
+			keyData, err := ioutil.ReadFile(keyPath)
+			if err != nil {
+				return nil, err
+			}
+			*file.key = keyData
+		}
+	}
+	return r, nil
+}
+
 func (r *RawTLSAssets) WriteToDir(dirname string, includeCAKey bool) error {
 	assets := []struct {
 		name      string
@@ -241,11 +292,11 @@ type EncryptService interface {
 	Encrypt(*kms.EncryptInput) (*kms.EncryptOutput, error)
 }
 
-func (r *RawTLSAssets) Compact(kMSKeyARN string, kmsSvc EncryptService) (*CompactTLSAssets, error) {
+func (r *RawTLSAssets) Encrypt(kMSKeyARN string, kmsSvc EncryptService) (*EncryptedTLSAssets, error) {
 	var err error
-	compact := func(data []byte) string {
+	encrypt := func(data []byte) []byte {
 		if err != nil {
-			return ""
+			return []byte{}
 		}
 
 		encryptInput := kms.EncryptInput{
@@ -255,9 +306,64 @@ func (r *RawTLSAssets) Compact(kMSKeyARN string, kmsSvc EncryptService) (*Compac
 
 		var encryptOutput *kms.EncryptOutput
 		if encryptOutput, err = kmsSvc.Encrypt(&encryptInput); err != nil {
+			return []byte{}
+		}
+		return encryptOutput.CiphertextBlob
+	}
+	encryptedAssets := EncryptedTLSAssets{
+		CACert:         encrypt(r.CACert),
+		APIServerCert:  encrypt(r.APIServerCert),
+		APIServerKey:   encrypt(r.APIServerKey),
+		WorkerCert:     encrypt(r.WorkerCert),
+		WorkerKey:      encrypt(r.WorkerKey),
+		AdminCert:      encrypt(r.AdminCert),
+		AdminKey:       encrypt(r.AdminKey),
+		EtcdCert:       encrypt(r.EtcdCert),
+		EtcdClientCert: encrypt(r.EtcdClientCert),
+		EtcdClientKey:  encrypt(r.EtcdClientKey),
+		EtcdKey:        encrypt(r.EtcdKey),
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &encryptedAssets, nil
+}
+
+func (r *EncryptedTLSAssets) WriteToDir(dirname string, includeCAKey bool) error {
+	assets := []struct {
+		name      string
+		cert, key []byte
+	}{
+		{"ca", r.CACert, r.CAKey},
+		{"apiserver", r.APIServerCert, r.APIServerKey},
+		{"worker", r.WorkerCert, r.WorkerKey},
+		{"admin", r.AdminCert, r.AdminKey},
+		{"etcd", r.EtcdCert, r.EtcdKey},
+		{"etcd-client", r.EtcdClientCert, r.EtcdClientKey},
+	}
+	for _, asset := range assets {
+		certPath := filepath.Join(dirname, asset.name+".pem.enc")
+		keyPath := filepath.Join(dirname, asset.name+"-key.pem.enc")
+
+		if err := ioutil.WriteFile(certPath, asset.cert, 0600); err != nil {
+			return err
+		}
+
+		if asset.name != "ca" || includeCAKey {
+			if err := ioutil.WriteFile(keyPath, asset.key, 0600); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (r *EncryptedTLSAssets) Compact() (*CompactTLSAssets, error) {
+	var err error
+	compact := func(data []byte) string {
+		if err != nil {
 			return ""
 		}
-		data = encryptOutput.CiphertextBlob
 
 		var out string
 		if out, err = gzipcompressor.CompressData(data); err != nil {
@@ -283,3 +389,55 @@ func (r *RawTLSAssets) Compact(kMSKeyARN string, kmsSvc EncryptService) (*Compac
 	}
 	return &compactAssets, nil
 }
+
+type KMSConfig struct {
+	Region         string
+	EncryptService EncryptService
+	KMSKeyARN      string
+}
+
+func ReadOrCreateEncryptedTLSAssets(tlsAssetsDir string, kmsConfig KMSConfig) (*EncryptedTLSAssets, error) {
+	var kmsSvc EncryptService
+
+	encryptedAssets, err := ReadEncryptedTLSAssets(tlsAssetsDir)
+	if err != nil {
+		rawAssets, err := ReadRawTLSAssets(tlsAssetsDir)
+		if err != nil {
+			return nil, err
+		}
+
+		awsConfig := aws.NewConfig().
+			WithRegion(kmsConfig.Region).
+			WithCredentialsChainVerboseErrors(true)
+
+		// TODO Cleaner way to inject this dependency
+		if kmsConfig.EncryptService == nil {
+			kmsSvc = kms.New(session.New(awsConfig))
+		} else {
+			kmsSvc = kmsConfig.EncryptService
+		}
+
+		encryptedAssets, err = rawAssets.Encrypt(kmsConfig.KMSKeyARN, kmsSvc)
+		if err != nil {
+			return nil, fmt.Errorf("failed to encrypt TLS assets: %v", err)
+		}
+
+		// The fact KMS encryption produces different ciphertexts for the same plaintext had been
+		// causing unnecessary node replacements(https://github.com/coreos/kube-aws/issues/107)
+		// Write encrypted tls assets for caching purpose so that we can avoid that.
+		encryptedAssets.WriteToDir(tlsAssetsDir, true)
+	}
+
+	return encryptedAssets, nil
+}
+
+func ReadOrCreateCompactTLSAssets(tlsAssetsDir string, kmsConfig KMSConfig) (*CompactTLSAssets, error) {
+	encryptedAssets, err := ReadOrCreateEncryptedTLSAssets(tlsAssetsDir, kmsConfig)
+
+	compactAssets, err := encryptedAssets.Compact()
+	if err != nil {
+		return nil, fmt.Errorf("failed to compress TLS assets: %v", err)
+	}
+
+	return compactAssets, nil
+}

config/tls_config_test.go

@@ -6,6 +6,10 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"encoding/pem"
+	"github.com/coreos/kube-aws/test/helper"
+	"os"
+	"path/filepath"
+	"reflect"
 )
 
 func genTLSAssets(t *testing.T) *RawTLSAssets {
@@ -102,3 +106,112 @@ func TestTLSGeneration(t *testing.T) {
 		}
 	}
 }
+
+func TestReadOrCreateCompactTLSAssets(t *testing.T) {
+	helper.WithDummyCredentials(func(dir string) {
+		kmsConfig := KMSConfig{
+			KMSKeyARN:      "keyarn",
+			Region:         "us-west-1",
+			EncryptService: &dummyEncryptService{},
+		}
+
+		// See https://github.com/coreos/kube-aws/issues/107
+		t.Run("CachedToPreventUnnecessaryNodeReplacement", func(t *testing.T) {
+			created, err := ReadOrCreateCompactTLSAssets(dir, kmsConfig)
+
+			if err != nil {
+				t.Errorf("failed to read or update compact tls assets in %s : %v", dir, err)
+			}
+
+			// This depends on TestDummyEncryptService which ensures dummy encrypt service to produce different ciphertext for each encryption
+			// created == read means that encrypted assets were loaded from cached files named *.pem.enc, instead of re-encryptiong raw tls assets named *.pem files
+			// TODO Use some kind of mocking framework for tests like this
+			read, err := ReadOrCreateCompactTLSAssets(dir, kmsConfig)
+
+			if err != nil {
+				t.Errorf("failed to read or update compact tls assets in %s : %v", dir, err)
+			}
+
+			if !reflect.DeepEqual(created, read) {
+				t.Errorf(`failed to cache encrypted tls assets.
+	encrypted tls assets must not change after their first creation but they did change:
+	created = %v
+	read = %v`, created, read)
+			}
+		})
+
+		t.Run("RemoveOneOrMoreCacheFilesToRegenerateAll", func(t *testing.T) {
+			original, err := ReadOrCreateCompactTLSAssets(dir, kmsConfig)
+
+			if err != nil {
+				t.Errorf("failed to read the original encrypted tls assets : %v", err)
+			}
+
+			if err := os.Remove(filepath.Join(dir, "ca.pem.enc")); err != nil {
+				t.Errorf("failed to remove ca.pem.enc for test setup : %v", err)
+				t.FailNow()
+			}
+
+			regenerated, err := ReadOrCreateCompactTLSAssets(dir, kmsConfig)
+
+			if err != nil {
+				t.Errorf("failed to read the regenerated encrypted tls assets : %v", err)
+			}
+
+			if original.AdminCert == regenerated.AdminCert {
+				t.Errorf("AdminCert must change but it didn't : original = %v, regenrated = %v ", original.AdminCert, regenerated.AdminCert)
+			}
+
+			if original.AdminKey == regenerated.AdminKey {
+				t.Errorf("AdminKey must change but it didn't : original = %v, regenrated = %v ", original.AdminKey, regenerated.AdminKey)
+			}
+
+			if original.CACert == regenerated.CACert {
+				t.Errorf("CACert must change but it didn't : original = %v, regenrated = %v ", original.CACert, regenerated.CACert)
+			}
+
+			if original.CACert == regenerated.CACert {
+				t.Errorf("CACert must change but it didn't : original = %v, regenrated = %v ", original.CACert, regenerated.CACert)
+			}
+
+			if original.WorkerCert == regenerated.WorkerCert {
+				t.Errorf("WorkerCert must change but it didn't : original = %v, regenrated = %v ", original.WorkerCert, regenerated.WorkerCert)
+			}
+
+			if original.WorkerCert == regenerated.WorkerCert {
+				t.Errorf("WorkerCert must change but it didn't : original = %v, regenrated = %v ", original.WorkerCert, regenerated.WorkerCert)
+			}
+
+			if original.APIServerCert == regenerated.APIServerCert {
+				t.Errorf("APIServerCert must change but it didn't : original = %v, regenrated = %v ", original.APIServerCert, regenerated.APIServerCert)
+			}
+
+			if original.APIServerCert == regenerated.APIServerCert {
+				t.Errorf("APIServerCert must change but it didn't : original = %v, regenrated = %v ", original.APIServerCert, regenerated.APIServerCert)
+			}
+
+			if original.EtcdClientCert == regenerated.EtcdClientCert {
+				t.Errorf("EtcdClientCert must change but it didn't : original = %v, regenrated = %v ", original.EtcdClientCert, regenerated.EtcdClientCert)
+			}
+
+			if original.EtcdClientCert == regenerated.EtcdClientCert {
+				t.Errorf("EtcdClientCert must change but it didn't : original = %v, regenrated = %v ", original.EtcdClientCert, regenerated.EtcdClientCert)
+			}
+
+			if original.EtcdCert == regenerated.EtcdCert {
+				t.Errorf("EtcdCert must change but it didn't : original = %v, regenrated = %v ", original.EtcdCert, regenerated.EtcdCert)
+			}
+
+			if original.EtcdCert == regenerated.EtcdCert {
+				t.Errorf("EtcdCert must change but it didn't : original = %v, regenrated = %v ", original.EtcdCert, regenerated.EtcdCert)
+			}
+
+			if reflect.DeepEqual(original, regenerated) {
+				t.Errorf(`unexpecteed data contained in (possibly) regenerated encrypted tls assets.
+	encrypted tls assets must change after regeneration but they didn't:
+	original = %v
+	regenerated = %v`, original, regenerated)
+			}
+		})
+	})
+}