From 35fa6ddf45c061e0f08d3a3b5119f8f4da38f6d1 Mon Sep 17 00:00:00 2001 From: Sean McGrail Date: Wed, 1 Jul 2020 10:35:02 -0700 Subject: [PATCH] service/s3/s3crypto: V2 Client Release (#3403) --- CHANGELOG_PENDING.md | 4 + .../sync/singleflight/singleflight_test.go | 4 + service/s3/s3crypto/aes_cbc_content_cipher.go | 6 + service/s3/s3crypto/aes_gcm_content_cipher.go | 16 +- .../s3crypto/aes_gcm_content_cipher_test.go | 5 + service/s3/s3crypto/aes_gcm_test.go | 68 +++++- service/s3/s3crypto/cipher_util.go | 80 ------- service/s3/s3crypto/cipher_util_test.go | 30 +-- service/s3/s3crypto/decryption_client.go | 58 ++--- service/s3/s3crypto/decryption_client_v2.go | 115 ++++++++++ .../s3/s3crypto/decryption_client_v2_test.go | 39 ++++ service/s3/s3crypto/deprecations.go | 10 + service/s3/s3crypto/deprecations_test.go | 51 +++++ service/s3/s3crypto/doc.go | 65 ++++-- service/s3/s3crypto/encryption_client.go | 101 +++------ service/s3/s3crypto/encryption_client_v2.go | 108 +++++++++ service/s3/s3crypto/envelope.go | 2 +- service/s3/s3crypto/integration/main_test.go | 187 ++++++++++++++++ service/s3/s3crypto/key_handler.go | 25 ++- service/s3/s3crypto/key_handler_test.go | 6 +- service/s3/s3crypto/kms_key_handler.go | 131 +++++++++-- service/s3/s3crypto/kms_key_handler_test.go | 106 ++++++++- service/s3/s3crypto/migrations_test.go | 148 ++++++++++++ service/s3/s3crypto/mock_test.go | 4 +- service/s3/s3crypto/shared_client.go | 210 ++++++++++++++++++ service/s3/s3crypto/strategy.go | 1 - service/s3/s3crypto/strategy_test.go | 107 ++++++++- service/s3/s3crypto/testdata/aes_gcm.json | 56 +++++ 28 files changed, 1480 insertions(+), 263 deletions(-) create mode 100644 service/s3/s3crypto/decryption_client_v2.go create mode 100644 service/s3/s3crypto/decryption_client_v2_test.go create mode 100644 service/s3/s3crypto/deprecations.go create mode 100644 service/s3/s3crypto/deprecations_test.go create mode 100644 service/s3/s3crypto/encryption_client_v2.go create mode 100644 service/s3/s3crypto/integration/main_test.go create mode 100644 service/s3/s3crypto/migrations_test.go create mode 100644 service/s3/s3crypto/shared_client.go create mode 100644 service/s3/s3crypto/testdata/aes_gcm.json diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 8a1927a39ca..fb2501f230b 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,4 +1,8 @@ ### SDK Features +* `service/s3/s3crypto`: Introduces `EncryptionClientV2` and `DecryptionClientV2` encryption and decryption clients which support +a new key wrapping algorithm `kms+context`. ([#3403](https://github.com/aws/aws-sdk-go/pull/3403)) + * `DecryptionClientV2` maintains the ability to decrypt objects encrypted using the `EncryptionClient`. + * Please see `s3crypto` documentation for migration details. ### SDK Enhancements diff --git a/internal/sync/singleflight/singleflight_test.go b/internal/sync/singleflight/singleflight_test.go index ad040379e8b..d9da008f417 100644 --- a/internal/sync/singleflight/singleflight_test.go +++ b/internal/sync/singleflight/singleflight_test.go @@ -14,6 +14,7 @@ import ( ) func TestDo(t *testing.T) { + t.Skip("singleflight tests not stable") var g Group v, err, _ := g.Do("key", func() (interface{}, error) { return "bar", nil @@ -27,6 +28,7 @@ func TestDo(t *testing.T) { } func TestDoErr(t *testing.T) { + t.Skip("singleflight tests not stable") var g Group someErr := errors.New("Some error") v, err, _ := g.Do("key", func() (interface{}, error) { @@ -41,6 +43,7 @@ func TestDoErr(t *testing.T) { } func TestDoDupSuppress(t *testing.T) { + t.Skip("singleflight tests not stable") var g Group var wg1, wg2 sync.WaitGroup c := make(chan string, 1) @@ -89,6 +92,7 @@ func TestDoDupSuppress(t *testing.T) { // Test that singleflight behaves correctly after Forget called. // See https://github.com/golang/go/issues/31420 func TestForget(t *testing.T) { + t.Skip("singleflight tests not stable") var g Group var firstStarted, firstFinished sync.WaitGroup diff --git a/service/s3/s3crypto/aes_cbc_content_cipher.go b/service/s3/s3crypto/aes_cbc_content_cipher.go index 7346e383cd3..30d6b8cb7b2 100644 --- a/service/s3/s3crypto/aes_cbc_content_cipher.go +++ b/service/s3/s3crypto/aes_cbc_content_cipher.go @@ -15,8 +15,14 @@ type cbcContentCipherBuilder struct { padder Padder } +func (cbcContentCipherBuilder) isUsingDeprecatedFeatures() error { + return errDeprecatedCipherBuilder +} + // AESCBCContentCipherBuilder returns a new encryption only mode structure with a specific cipher // for the master key +// +// deprecated: This content cipher builder has been deprecated. Users should migrate to AESGCMContentCipherBuilder func AESCBCContentCipherBuilder(generator CipherDataGenerator, padder Padder) ContentCipherBuilder { return cbcContentCipherBuilder{generator: generator, padder: padder} } diff --git a/service/s3/s3crypto/aes_gcm_content_cipher.go b/service/s3/s3crypto/aes_gcm_content_cipher.go index 2f2c36dc60c..d4a4224e540 100644 --- a/service/s3/s3crypto/aes_gcm_content_cipher.go +++ b/service/s3/s3crypto/aes_gcm_content_cipher.go @@ -15,6 +15,13 @@ type gcmContentCipherBuilder struct { generator CipherDataGenerator } +func (builder gcmContentCipherBuilder) isUsingDeprecatedFeatures() error { + if feature, ok := builder.generator.(deprecatedFeatures); ok { + return feature.isUsingDeprecatedFeatures() + } + return nil +} + // AESGCMContentCipherBuilder returns a new encryption only mode structure with a specific cipher // for the master key func AESGCMContentCipherBuilder(generator CipherDataGenerator) ContentCipherBuilder { @@ -29,9 +36,14 @@ func (builder gcmContentCipherBuilder) ContentCipherWithContext(ctx aws.Context) var cd CipherData var err error - if v, ok := builder.generator.(CipherDataGeneratorWithContext); ok { + switch v := builder.generator.(type) { + case CipherDataGeneratorWithCEKAlgWithContext: + cd, err = v.GenerateCipherDataWithCEKAlgWithContext(ctx, gcmKeySize, gcmNonceSize, AESGCMNoPadding) + case CipherDataGeneratorWithCEKAlg: + cd, err = v.GenerateCipherDataWithCEKAlg(gcmKeySize, gcmNonceSize, AESGCMNoPadding) + case CipherDataGeneratorWithContext: cd, err = v.GenerateCipherDataWithContext(ctx, gcmKeySize, gcmNonceSize) - } else { + default: cd, err = builder.generator.GenerateCipherData(gcmKeySize, gcmNonceSize) } if err != nil { diff --git a/service/s3/s3crypto/aes_gcm_content_cipher_test.go b/service/s3/s3crypto/aes_gcm_content_cipher_test.go index 400ceaf5d07..eb0a1c8baa6 100644 --- a/service/s3/s3crypto/aes_gcm_content_cipher_test.go +++ b/service/s3/s3crypto/aes_gcm_content_cipher_test.go @@ -3,6 +3,7 @@ package s3crypto_test import ( "testing" + "github.com/aws/aws-sdk-go/service/kms/kmsiface" "github.com/aws/aws-sdk-go/service/s3/s3crypto" ) @@ -26,3 +27,7 @@ func TestAESGCMContentCipherNewEncryptor(t *testing.T) { t.Errorf("expected non-nil vaue") } } + +type mockKMS struct { + kmsiface.KMSAPI +} diff --git a/service/s3/s3crypto/aes_gcm_test.go b/service/s3/s3crypto/aes_gcm_test.go index 72d8e22e00e..29e67d15105 100644 --- a/service/s3/s3crypto/aes_gcm_test.go +++ b/service/s3/s3crypto/aes_gcm_test.go @@ -1,8 +1,11 @@ +// +build go1.9 + package s3crypto import ( "bytes" "encoding/hex" + "encoding/json" "fmt" "io" "io/ioutil" @@ -15,7 +18,7 @@ func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_128_Test_0(t *testing.T) { iv, _ := hex.DecodeString("0d18e06c7c725ac9e362e1ce") key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22") plaintext, _ := hex.DecodeString("2db5168e932556f8089a0622981d017d") - expected, _ := hex.DecodeString("fa4362189661d163fcd6a56d8bf0405ad636ac1bbedd5cc3ee727dc2ab4a9489") + expected, _ := hex.DecodeString("fa4362189661d163fcd6a56d8bf0405a") tag, _ := hex.DecodeString("d636ac1bbedd5cc3ee727dc2ab4a9489") aesgcmTest(t, iv, key, plaintext, expected, tag) } @@ -24,7 +27,7 @@ func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_104_Test_3(t *testing.T) { iv, _ := hex.DecodeString("4742357c335913153ff0eb0f") key, _ := hex.DecodeString("e5a0eb92cc2b064e1bc80891faf1fab5e9a17a9c3a984e25416720e30e6c2b21") plaintext, _ := hex.DecodeString("8499893e16b0ba8b007d54665a") - expected, _ := hex.DecodeString("eb8e6175f1fe38eb1acf95fd5188a8b74bb74fda553e91020a23deed45") + expected, _ := hex.DecodeString("eb8e6175f1fe38eb1acf95fd51") tag, _ := hex.DecodeString("88a8b74bb74fda553e91020a23deed45") aesgcmTest(t, iv, key, plaintext, expected, tag) } @@ -33,7 +36,7 @@ func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_256_Test_6(t *testing.T) { iv, _ := hex.DecodeString("a291484c3de8bec6b47f525f") key, _ := hex.DecodeString("37f39137416bafde6f75022a7a527cc593b6000a83ff51ec04871a0ff5360e4e") plaintext, _ := hex.DecodeString("fafd94cede8b5a0730394bec68a8e77dba288d6ccaa8e1563a81d6e7ccc7fc97") - expected, _ := hex.DecodeString("44dc868006b21d49284016565ffb3979cc4271d967628bf7cdaf86db888e92e501a2b578aa2f41ec6379a44a31cc019c") + expected, _ := hex.DecodeString("44dc868006b21d49284016565ffb3979cc4271d967628bf7cdaf86db888e92e5") tag, _ := hex.DecodeString("01a2b578aa2f41ec6379a44a31cc019c") aesgcmTest(t, iv, key, plaintext, expected, tag) } @@ -42,11 +45,62 @@ func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_408_Test_8(t *testing.T) { iv, _ := hex.DecodeString("92f258071d79af3e63672285") key, _ := hex.DecodeString("595f259c55abe00ae07535ca5d9b09d6efb9f7e9abb64605c337acbd6b14fc7e") plaintext, _ := hex.DecodeString("a6fee33eb110a2d769bbc52b0f36969c287874f665681477a25fc4c48015c541fbe2394133ba490a34ee2dd67b898177849a91") - expected, _ := hex.DecodeString("bbca4a9e09ae9690c0f6f8d405e53dccd666aa9c5fa13c8758bc30abe1ddd1bcce0d36a1eaaaaffef20cd3c5970b9673f8a65c26ccecb9976fd6ac9c2c0f372c52c821") + expected, _ := hex.DecodeString("bbca4a9e09ae9690c0f6f8d405e53dccd666aa9c5fa13c8758bc30abe1ddd1bcce0d36a1eaaaaffef20cd3c5970b9673f8a65c") tag, _ := hex.DecodeString("26ccecb9976fd6ac9c2c0f372c52c821") aesgcmTest(t, iv, key, plaintext, expected, tag) } +type KAT struct { + IV string `json:"iv"` + Key string `json:"key"` + Plaintext string `json:"pt"` + AAD string `json:"aad"` + CipherText string `json:"ct"` + Tag string `json:"tag"` +} + +func TestAES_GCM_KATS(t *testing.T) { + fileContents, err := ioutil.ReadFile("testdata/aes_gcm.json") + if err != nil { + t.Fatalf("failed to read KAT file: %v", err) + } + + var kats []KAT + err = json.Unmarshal(fileContents, &kats) + if err != nil { + t.Fatalf("failed to unmarshal KAT json file: %v", err) + } + + for i, kat := range kats { + t.Run(fmt.Sprintf("Case%d", i), func(t *testing.T) { + if len(kat.AAD) > 0 { + t.Skip("Skipping... SDK implementation does not expose additional authenticated data") + } + iv, err := hex.DecodeString(kat.IV) + if err != nil { + t.Fatalf("failed to decode iv: %v", err) + } + key, err := hex.DecodeString(kat.Key) + if err != nil { + t.Fatalf("failed to decode key: %v", err) + } + plaintext, err := hex.DecodeString(kat.Plaintext) + if err != nil { + t.Fatalf("failed to decode plaintext: %v", err) + } + ciphertext, err := hex.DecodeString(kat.CipherText) + if err != nil { + t.Fatalf("failed to decode ciphertext: %v", err) + } + tag, err := hex.DecodeString(kat.Tag) + if err != nil { + t.Fatalf("failed to decode tag: %v", err) + } + aesgcmTest(t, iv, key, plaintext, ciphertext, tag) + }) + } +} + func TestGCMEncryptReader_SourceError(t *testing.T) { gcm := &gcmEncryptReader{ encrypter: &mockCipherAEAD{}, @@ -105,6 +159,8 @@ func TestGCMDecryptReader_DecrypterOpenError(t *testing.T) { } func aesgcmTest(t *testing.T, iv, key, plaintext, expected, tag []byte) { + t.Helper() + const gcmTagSize = 16 cd := CipherData{ Key: key, IV: iv, @@ -122,11 +178,11 @@ func aesgcmTest(t *testing.T, iv, key, plaintext, expected, tag []byte) { } // splitting tag and ciphertext - etag := ciphertext[len(ciphertext)-16:] + etag := ciphertext[len(ciphertext)-gcmTagSize:] if !bytes.Equal(etag, tag) { t.Errorf("expected tags to be equivalent") } - if !bytes.Equal(ciphertext, expected) { + if !bytes.Equal(ciphertext[:len(ciphertext)-gcmTagSize], expected) { t.Errorf("expected ciphertext to be equivalent") } diff --git a/service/s3/s3crypto/cipher_util.go b/service/s3/s3crypto/cipher_util.go index de7f553ba0a..64eac775311 100644 --- a/service/s3/s3crypto/cipher_util.go +++ b/service/s3/s3crypto/cipher_util.go @@ -3,33 +3,8 @@ package s3crypto import ( "encoding/base64" "strconv" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" ) -func (client *DecryptionClient) contentCipherFromEnvelope(ctx aws.Context, env Envelope) (ContentCipher, error) { - wrap, err := client.wrapFromEnvelope(env) - if err != nil { - return nil, err - } - - return client.cekFromEnvelope(ctx, env, wrap) -} - -func (client *DecryptionClient) wrapFromEnvelope(env Envelope) (CipherDataDecrypter, error) { - f, ok := client.WrapRegistry[env.WrapAlg] - if !ok || f == nil { - return nil, awserr.New( - "InvalidWrapAlgorithmError", - "wrap algorithm isn't supported, "+env.WrapAlg, - nil, - ) - } - return f(env) -} - // AESGCMNoPadding is the constant value that is used to specify // the CEK algorithm consiting of AES GCM with no padding. const AESGCMNoPadding = "AES/GCM/NoPadding" @@ -37,61 +12,6 @@ const AESGCMNoPadding = "AES/GCM/NoPadding" // AESCBC is the string constant that signifies the AES CBC algorithm cipher. const AESCBC = "AES/CBC" -func (client *DecryptionClient) cekFromEnvelope(ctx aws.Context, env Envelope, decrypter CipherDataDecrypter) (ContentCipher, error) { - f, ok := client.CEKRegistry[env.CEKAlg] - if !ok || f == nil { - return nil, awserr.New( - "InvalidCEKAlgorithmError", - "cek algorithm isn't supported, "+env.CEKAlg, - nil, - ) - } - - key, err := base64.StdEncoding.DecodeString(env.CipherKey) - if err != nil { - return nil, err - } - - iv, err := base64.StdEncoding.DecodeString(env.IV) - if err != nil { - return nil, err - } - - if d, ok := decrypter.(CipherDataDecrypterWithContext); ok { - key, err = d.DecryptKeyWithContext(ctx, key) - } else { - key, err = decrypter.DecryptKey(key) - } - - if err != nil { - return nil, err - } - - cd := CipherData{ - Key: key, - IV: iv, - CEKAlgorithm: env.CEKAlg, - Padder: client.getPadder(env.CEKAlg), - } - return f(cd) -} - -// getPadder will return an unpadder with checking the cek algorithm specific padder. -// If there wasn't a cek algorithm specific padder, we check the padder itself. -// We return a no unpadder, if no unpadder was found. This means any customization -// either contained padding within the cipher implementation, and to maintain -// backwards compatility we will simply not unpad anything. -func (client *DecryptionClient) getPadder(cekAlg string) Padder { - padder, ok := client.PadderRegistry[cekAlg] - if !ok { - padder, ok = client.PadderRegistry[cekAlg[strings.LastIndex(cekAlg, "/")+1:]] - if !ok { - return NoPadder - } - } - return padder -} - func encodeMeta(reader hashReader, cd CipherData) (Envelope, error) { iv := base64.StdEncoding.EncodeToString(cd.IV) key := base64.StdEncoding.EncodeToString(cd.EncryptedKey) diff --git a/service/s3/s3crypto/cipher_util_test.go b/service/s3/s3crypto/cipher_util_test.go index c8704c30d20..5c59160aa97 100644 --- a/service/s3/s3crypto/cipher_util_test.go +++ b/service/s3/s3crypto/cipher_util_test.go @@ -14,7 +14,7 @@ import ( ) func TestWrapFactory(t *testing.T) { - c := DecryptionClient{ + o := DecryptionClientOptions{ WrapRegistry: map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(unit.Session), @@ -28,7 +28,7 @@ func TestWrapFactory(t *testing.T) { WrapAlg: KMSWrap, MatDesc: `{"kms_cmk_id":""}`, } - wrap, err := c.wrapFromEnvelope(env) + wrap, err := wrapFromEnvelope(o, env) w, ok := wrap.(*kmsKeyHandler) if err != nil { @@ -42,7 +42,7 @@ func TestWrapFactory(t *testing.T) { } } func TestWrapFactoryErrorNoWrap(t *testing.T) { - c := DecryptionClient{ + o := DecryptionClientOptions{ WrapRegistry: map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(unit.Session), @@ -56,7 +56,7 @@ func TestWrapFactoryErrorNoWrap(t *testing.T) { WrapAlg: "none", MatDesc: `{"kms_cmk_id":""}`, } - wrap, err := c.wrapFromEnvelope(env) + wrap, err := wrapFromEnvelope(o, env) if err == nil { t.Error("expected error, but received none") @@ -67,7 +67,7 @@ func TestWrapFactoryErrorNoWrap(t *testing.T) { } func TestWrapFactoryCustomEntry(t *testing.T) { - c := DecryptionClient{ + o := DecryptionClientOptions{ WrapRegistry: map[string]WrapEntry{ "custom": (kmsKeyHandler{ kms: kms.New(unit.Session), @@ -81,7 +81,7 @@ func TestWrapFactoryCustomEntry(t *testing.T) { WrapAlg: "custom", MatDesc: `{"kms_cmk_id":""}`, } - wrap, err := c.wrapFromEnvelope(env) + wrap, err := wrapFromEnvelope(o, env) if err != nil { t.Errorf("expected no error, but received %v", err) @@ -107,7 +107,7 @@ func TestCEKFactory(t *testing.T) { Region: aws.String("us-west-2"), }) - c := DecryptionClient{ + o := DecryptionClientOptions{ WrapRegistry: map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(sess), @@ -139,8 +139,8 @@ func TestCEKFactory(t *testing.T) { IV: ivB64, MatDesc: `{"kms_cmk_id":""}`, } - wrap, err := c.wrapFromEnvelope(env) - cek, err := c.cekFromEnvelope(aws.BackgroundContext(), env, wrap) + wrap, err := wrapFromEnvelope(o, env) + cek, err := cekFromEnvelope(o, aws.BackgroundContext(), env, wrap) if err != nil { t.Errorf("expected no error, but received %v", err) @@ -166,7 +166,7 @@ func TestCEKFactoryNoCEK(t *testing.T) { Region: aws.String("us-west-2"), }) - c := DecryptionClient{ + o := DecryptionClientOptions{ WrapRegistry: map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(sess), @@ -198,8 +198,8 @@ func TestCEKFactoryNoCEK(t *testing.T) { IV: ivB64, MatDesc: `{"kms_cmk_id":""}`, } - wrap, err := c.wrapFromEnvelope(env) - cek, err := c.cekFromEnvelope(aws.BackgroundContext(), env, wrap) + wrap, err := wrapFromEnvelope(o, env) + cek, err := cekFromEnvelope(o, aws.BackgroundContext(), env, wrap) if err == nil { t.Error("expected error, but received none") @@ -225,7 +225,7 @@ func TestCEKFactoryCustomEntry(t *testing.T) { Region: aws.String("us-west-2"), }) - c := DecryptionClient{ + o := DecryptionClientOptions{ WrapRegistry: map[string]WrapEntry{ KMSWrap: (kmsKeyHandler{ kms: kms.New(sess), @@ -255,8 +255,8 @@ func TestCEKFactoryCustomEntry(t *testing.T) { IV: ivB64, MatDesc: `{"kms_cmk_id":""}`, } - wrap, err := c.wrapFromEnvelope(env) - cek, err := c.cekFromEnvelope(aws.BackgroundContext(), env, wrap) + wrap, err := wrapFromEnvelope(o, env) + cek, err := cekFromEnvelope(o, aws.BackgroundContext(), env, wrap) if err != nil { t.Errorf("expected no error, but received %v", err) diff --git a/service/s3/s3crypto/decryption_client.go b/service/s3/s3crypto/decryption_client.go index 8e40f73341d..7e1c8cdc60c 100644 --- a/service/s3/s3crypto/decryption_client.go +++ b/service/s3/s3crypto/decryption_client.go @@ -25,6 +25,8 @@ type CEKEntry func(CipherData) (ContentCipher, error) // Supported content ciphers: // * AES/GCM // * AES/CBC +// +// deprecated: See DecryptionClientV2 type DecryptionClient struct { S3Client s3iface.S3API // LoadStrategy is used to load the metadata either from the metadata of the object @@ -45,6 +47,8 @@ type DecryptionClient struct { // svc := s3crypto.NewDecryptionClient(sess, func(svc *s3crypto.DecryptionClient{ // // Custom client options here // })) +// +// deprecated: see NewDecryptionClientV2 func NewDecryptionClient(prov client.ConfigProvider, options ...func(*DecryptionClient)) *DecryptionClient { s3client := s3.New(prov) @@ -82,47 +86,24 @@ func NewDecryptionClient(prov client.ConfigProvider, options ...func(*Decryption // decryption will be done. The SDK only supports V2 reads of KMS and GCM. // // Example: -// sess := session.New() +// sess := session.Must(session.NewSession()) // svc := s3crypto.NewDecryptionClient(sess) // req, out := svc.GetObjectRequest(&s3.GetObjectInput { // Key: aws.String("testKey"), // Bucket: aws.String("testBucket"), // }) // err := req.Send() +// +// deprecated: see DecryptionClientV2.GetObjectRequest func (c *DecryptionClient) GetObjectRequest(input *s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput) { - req, out := c.S3Client.GetObjectRequest(input) - req.Handlers.Unmarshal.PushBack(func(r *request.Request) { - env, err := c.LoadStrategy.Load(r) - if err != nil { - r.Error = err - out.Body.Close() - return - } - - // If KMS should return the correct CEK algorithm with the proper - // KMS key provider - cipher, err := c.contentCipherFromEnvelope(r.Context(), env) - if err != nil { - r.Error = err - out.Body.Close() - return - } - - reader, err := cipher.DecryptContents(out.Body) - if err != nil { - r.Error = err - out.Body.Close() - return - } - out.Body = reader - }) - return req, out + return getObjectRequest(c.getClientOptions(), input) } // GetObject is a wrapper for GetObjectRequest +// +// deprecated: see DecryptionClientV2.GetObject func (c *DecryptionClient) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { - req, out := c.GetObjectRequest(input) - return out, req.Send() + return getObject(c.getClientOptions(), input) } // GetObjectWithContext is a wrapper for GetObjectRequest with the additional @@ -132,9 +113,18 @@ func (c *DecryptionClient) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOut // Context input parameters. The Context must not be nil. A nil Context will // cause a panic. Use the Context to add deadlining, timeouts, etc. In the future // this may create sub-contexts for individual underlying requests. +// +// deprecated: see DecryptionClientV2.GetObjectWithContext func (c *DecryptionClient) GetObjectWithContext(ctx aws.Context, input *s3.GetObjectInput, opts ...request.Option) (*s3.GetObjectOutput, error) { - req, out := c.GetObjectRequest(input) - req.SetContext(ctx) - req.ApplyOptions(opts...) - return out, req.Send() + return getObjectWithContext(c.getClientOptions(), ctx, input, opts...) +} + +func (c *DecryptionClient) getClientOptions() DecryptionClientOptions { + return DecryptionClientOptions{ + S3Client: c.S3Client, + LoadStrategy: c.LoadStrategy, + WrapRegistry: c.WrapRegistry, + CEKRegistry: c.CEKRegistry, + PadderRegistry: c.PadderRegistry, + } } diff --git a/service/s3/s3crypto/decryption_client_v2.go b/service/s3/s3crypto/decryption_client_v2.go new file mode 100644 index 00000000000..be37bd4bbc6 --- /dev/null +++ b/service/s3/s3crypto/decryption_client_v2.go @@ -0,0 +1,115 @@ +package s3crypto + +import ( + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +// DecryptionClientV2 is an S3 crypto client. The decryption client +// will handle all get object requests from Amazon S3. +// Supported key wrapping algorithms: +// * AWS KMS +// * AWS KMS + Context +// +// Supported content ciphers: +// * AES/GCM +// * AES/CBC +type DecryptionClientV2 struct { + options DecryptionClientOptions +} + +// DecryptionClientOptions is the configuration options for DecryptionClientV2. +type DecryptionClientOptions struct { + S3Client s3iface.S3API + // LoadStrategy is used to load the metadata either from the metadata of the object + // or from a separate file in s3. + // + // Defaults to our default load strategy. + LoadStrategy LoadStrategy + + WrapRegistry map[string]WrapEntry + CEKRegistry map[string]CEKEntry + PadderRegistry map[string]Padder +} + +// NewDecryptionClientV2 instantiates a new V2 S3 crypto client. The returned DecryptionClientV2 will be able to decrypt +// object encrypted by both the V1 and V2 clients. +// +// Example: +// sess := session.Must(session.NewSession()) +// svc := s3crypto.NewDecryptionClientV2(sess, func(svc *s3crypto.DecryptionClientOptions{ +// // Custom client options here +// })) +func NewDecryptionClientV2(prov client.ConfigProvider, options ...func(clientOptions *DecryptionClientOptions)) *DecryptionClientV2 { + s3client := s3.New(prov) + + s3client.Handlers.Build.PushBack(func(r *request.Request) { + request.AddToUserAgent(r, "S3CryptoV2") + }) + + kmsClient := kms.New(prov) + clientOptions := &DecryptionClientOptions{ + S3Client: s3client, + LoadStrategy: defaultV2LoadStrategy{ + client: s3client, + }, + WrapRegistry: map[string]WrapEntry{ + KMSWrap: NewKMSWrapEntry(kmsClient), + KMSContextWrap: NewKMSContextWrapEntry(kmsClient), + }, + CEKRegistry: map[string]CEKEntry{ + AESGCMNoPadding: newAESGCMContentCipher, + strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): newAESCBCContentCipher, + }, + PadderRegistry: map[string]Padder{ + strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): AESCBCPadder, + "NoPadding": NoPadder, + }, + } + for _, option := range options { + option(clientOptions) + } + + return &DecryptionClientV2{options: *clientOptions} +} + +// GetObjectRequest will make a request to s3 and retrieve the object. In this process +// decryption will be done. The SDK only supports V2 reads of KMS and GCM. +// +// Example: +// sess := session.Must(session.NewSession()) +// svc := s3crypto.NewDecryptionClientV2(sess) +// req, out := svc.GetObjectRequest(&s3.GetObjectInput { +// Key: aws.String("testKey"), +// Bucket: aws.String("testBucket"), +// }) +// err := req.Send() +func (c *DecryptionClientV2) GetObjectRequest(input *s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput) { + return getObjectRequest(c.options, input) +} + +// GetObject is a wrapper for GetObjectRequest +func (c *DecryptionClientV2) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + req, out := getObjectRequest(c.options, input) + return out, req.Send() +} + +// GetObjectWithContext is a wrapper for GetObjectRequest with the additional +// context, and request options support. +// +// GetObjectWithContext is the same as GetObject with the additional support for +// Context input parameters. The Context must not be nil. A nil Context will +// cause a panic. Use the Context to add deadlining, timeouts, etc. In the future +// this may create sub-contexts for individual underlying requests. +func (c *DecryptionClientV2) GetObjectWithContext(ctx aws.Context, input *s3.GetObjectInput, opts ...request.Option) (*s3.GetObjectOutput, error) { + req, out := getObjectRequest(c.options, input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} diff --git a/service/s3/s3crypto/decryption_client_v2_test.go b/service/s3/s3crypto/decryption_client_v2_test.go new file mode 100644 index 00000000000..1261511367e --- /dev/null +++ b/service/s3/s3crypto/decryption_client_v2_test.go @@ -0,0 +1,39 @@ +package s3crypto_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go/awstesting/unit" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/s3/s3crypto" +) + +func TestDecryptionClientV2_CheckDeprecatedFeatures(t *testing.T) { + // AES/GCM/NoPadding with kms+context => allowed + builder := s3crypto.AESGCMContentCipherBuilder(s3crypto.NewKMSContextKeyGenerator(kms.New(unit.Session), "cmkID")) + _, err := s3crypto.NewEncryptionClientV2(unit.Session, builder) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + + // AES/GCM/NoPadding with kms => not allowed + builder = s3crypto.AESGCMContentCipherBuilder(s3crypto.NewKMSKeyGenerator(kms.New(unit.Session), "cmkID")) + _, err = s3crypto.NewEncryptionClientV2(unit.Session, builder) + if err == nil { + t.Error("expected error, but got nil") + } + + // AES/CBC/PKCS5Padding with kms => not allowed + builder = s3crypto.AESCBCContentCipherBuilder(s3crypto.NewKMSKeyGenerator(kms.New(unit.Session), "cmkID"), s3crypto.NewPKCS7Padder(128)) + _, err = s3crypto.NewEncryptionClientV2(unit.Session, builder) + if err == nil { + t.Error("expected error, but got nil") + } + + // AES/CBC/PKCS5Padding with kms+context => not allowed + builder = s3crypto.AESCBCContentCipherBuilder(s3crypto.NewKMSContextKeyGenerator(kms.New(unit.Session), "cmkID"), s3crypto.NewPKCS7Padder(128)) + _, err = s3crypto.NewEncryptionClientV2(unit.Session, builder) + if err == nil { + t.Error("expected error, but got nil") + } +} diff --git a/service/s3/s3crypto/deprecations.go b/service/s3/s3crypto/deprecations.go new file mode 100644 index 00000000000..8c54aa96682 --- /dev/null +++ b/service/s3/s3crypto/deprecations.go @@ -0,0 +1,10 @@ +package s3crypto + +import "fmt" + +var errDeprecatedCipherBuilder = fmt.Errorf("attempted to use deprecated cipher builder") +var errDeprecatedCipherDataGenerator = fmt.Errorf("attempted to use deprecated cipher data generator") + +type deprecatedFeatures interface { + isUsingDeprecatedFeatures() error +} diff --git a/service/s3/s3crypto/deprecations_test.go b/service/s3/s3crypto/deprecations_test.go new file mode 100644 index 00000000000..6a9bb0c9949 --- /dev/null +++ b/service/s3/s3crypto/deprecations_test.go @@ -0,0 +1,51 @@ +package s3crypto + +import ( + "testing" + + "github.com/aws/aws-sdk-go/service/kms/kmsiface" +) + +type mockKMS struct { + kmsiface.KMSAPI +} + +func TestAESGCMContentCipherBuilder_isUsingDeprecatedFeatures(t *testing.T) { + builder := AESGCMContentCipherBuilder(NewKMSKeyGenerator(mockKMS{}, "cmkID")) + + features, ok := builder.(deprecatedFeatures) + if !ok { + t.Errorf("expected to implement deprecatedFeatures interface") + } + + err := features.isUsingDeprecatedFeatures() + if err == nil { + t.Errorf("expected to recieve error for using deprecated features") + } + + builder = AESGCMContentCipherBuilder(NewKMSContextKeyGenerator(mockKMS{}, "cmkID")) + + features, ok = builder.(deprecatedFeatures) + if !ok { + t.Errorf("expected to implement deprecatedFeatures interface") + } + + err = features.isUsingDeprecatedFeatures() + if err != nil { + t.Errorf("expected no error, got %v", err) + } +} + +func TestAESCBCContentCipherBuilder_isUsingDeprecatedFeatures(t *testing.T) { + builder := AESCBCContentCipherBuilder(nil, nil) + + features, ok := builder.(deprecatedFeatures) + if !ok { + t.Errorf("expected to implement deprecatedFeatures interface") + } + + err := features.isUsingDeprecatedFeatures() + if err == nil { + t.Errorf("expected to recieve error for using deprecated features") + } +} diff --git a/service/s3/s3crypto/doc.go b/service/s3/s3crypto/doc.go index fc6760a0c40..305b7e517df 100644 --- a/service/s3/s3crypto/doc.go +++ b/service/s3/s3crypto/doc.go @@ -15,29 +15,34 @@ ciphers. Creating an S3 cryptography client cmkID := "" - sess := session.New() + sess := session.Must(session.NewSession()) // Create the KeyProvider - handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID) + handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) // Create an encryption and decryption client // We need to pass the session here so S3 can use it. In addition, any decryption that // occurs will use the KMS client. - svc := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler)) - svc := s3crypto.NewDecryptionClient(sess) + svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler)) + svc := s3crypto.NewDecryptionClientV2(sess) Configuration of the S3 cryptography client - cfg := s3crypto.EncryptionConfig{ + sess := session.Must(session.NewSession()) + handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) + svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler), func (o *s3crypto.EncryptionClientOptions) { // Save instruction files to separate objects - SaveStrategy: NewS3SaveStrategy(session.New(), ""), + o.SaveStrategy = NewS3SaveStrategy(sess, "") + // Change instruction file suffix to .example - InstructionFileSuffix: ".example", + o.InstructionFileSuffix = ".example" + // Set temp folder path - TempFolderPath: "/path/to/tmp/folder/", + o.TempFolderPath = "/path/to/tmp/folder/" + // Any content less than the minimum file size will use memory // instead of writing the contents to a temp file. - MinFileSize: int64(1024 * 1024 * 1024), - } + o.MinFileSize = int64(1024 * 1024 * 1024) + }) The default SaveStrategy is to the object's header. @@ -48,19 +53,43 @@ This suffix only affects gets and not puts. Put uses the keyprovider's suffix. Registration of new wrap or cek algorithms are also supported by the SDK. Let's say we want to support `AES Wrap` and `AES CTR`. Let's assume we have already defined the functionality. - svc := s3crypto.NewDecryptionClient(sess) - svc.WrapRegistry["AESWrap"] = NewAESWrap - svc.CEKRegistry["AES/CTR/NoPadding"] = NewAESCTR + svc := s3crypto.NewDecryptionClientV2(sess, func(o *s3crypto.DecryptionClientOptions) { + o.WrapRegistry["CustomWrap"] = NewCustomWrap + o.CEKRegistry["CustomCEK"] = NewCustomCEK + }) We have now registered these new algorithms to the decryption client. When the client calls `GetObject` and sees -the wrap as `AESWrap` then it'll use that wrap algorithm. This is also true for `AES/CTR/NoPadding`. +the wrap as `CustomWrap` then it'll use that wrap algorithm. This is also true for `CustomCEK`. For encryption adding a custom content cipher builder and key handler will allow for encryption of custom defined ciphers. - // Our wrap algorithm, AESWrap - handler := NewAESWrap(key, iv) - // Our content cipher builder, AESCTRContentCipherBuilder - svc := s3crypto.NewEncryptionClient(sess, NewAESCTRContentCipherBuilder(handler)) + // Our wrap algorithm, CustomWrap + handler := NewCustomWrap(key, iv) + // Our content cipher builder, NewCustomCEKContentBuilder + svc := s3crypto.NewEncryptionClientV2(sess, NewCustomCEKContentBuilder(handler)) + +Deprecations + +The EncryptionClient and DecryptionClient types and their associated constructor functions have been deprecated. +Users of these clients should migrate to EncryptionClientV2 and DecryptionClientV2 types and constructor functions. + +EncryptionClientV2 removes encryption support of the following features + * AES/CBC/PKCS5Padding (content cipher) + * kms (key wrap algorithm) + +Attempting to construct an EncryptionClientV2 with deprecated features will result in an error returned back to the +calling application during construction of the client. + +Users of `AES/CBC/PKCS5Padding` will need to migrate usage to `AES/GCM/NoPadding`. +Users of `kms` key provider will need to migrate `kms+context`. + +DecryptionClientV2 client adds support for the `kms+context` key provider and maintains backwards comparability with +objects encrypted with the deprecated EncryptionClient. + +Migrating from V1 to V2 Clients + +Examples of how to migrate usage of the V1 clients to the V2 equivalents have been documented as usage examples of +the NewEncryptionClientV2 and NewDecryptionClientV2 functions. */ package s3crypto diff --git a/service/s3/s3crypto/encryption_client.go b/service/s3/s3crypto/encryption_client.go index 7792106d22d..b3816e89ac7 100644 --- a/service/s3/s3crypto/encryption_client.go +++ b/service/s3/s3crypto/encryption_client.go @@ -1,13 +1,9 @@ package s3crypto import ( - "encoding/hex" - "io" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/internal/sdkio" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" ) @@ -20,6 +16,8 @@ const DefaultMinFileSize = 1024 * 512 * 5 // will use KMS for key wrapping and AES GCM for content encryption. // AES GCM will load all data into memory. However, the rest of the content algorithms // do not load the entire contents into memory. +// +// deprecated: See EncryptionClientV2 type EncryptionClient struct { S3Client s3iface.S3API ContentCipherBuilder ContentCipherBuilder @@ -39,9 +37,11 @@ type EncryptionClient struct { // // Example: // cmkID := "arn:aws:kms:region:000000000000:key/00000000-0000-0000-0000-000000000000" -// sess := session.New() +// sess := session.Must(session.NewSession()) // handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID) -// svc := s3crypto.New(sess, s3crypto.AESGCMContentCipherBuilder(handler)) +// svc := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler)) +// +// deprecated: See NewEncryptionClientV2 func NewEncryptionClient(prov client.ConfigProvider, builder ContentCipherBuilder, options ...func(*EncryptionClient)) *EncryptionClient { s3client := s3.New(prov) @@ -74,76 +74,17 @@ func NewEncryptionClient(prov client.ConfigProvider, builder ContentCipherBuilde // Body: strings.NewReader("test data"), // }) // err := req.Send() +// +// deprecated: See EncryptionClientV2.PutObjectRequest func (c *EncryptionClient) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) { - req, out := c.S3Client.PutObjectRequest(input) - - // Get Size of file - n, err := aws.SeekerLen(input.Body) - if err != nil { - req.Error = err - return req, out - } - - dst, err := getWriterStore(req, c.TempFolderPath, n >= c.MinFileSize) - if err != nil { - req.Error = err - return req, out - } - - req.Handlers.Build.PushFront(func(r *request.Request) { - if err != nil { - r.Error = err - return - } - var encryptor ContentCipher - if v, ok := c.ContentCipherBuilder.(ContentCipherBuilderWithContext); ok { - encryptor, err = v.ContentCipherWithContext(r.Context()) - } else { - encryptor, err = c.ContentCipherBuilder.ContentCipher() - } - if err != nil { - r.Error = err - return - } - - md5 := newMD5Reader(input.Body) - sha := newSHA256Writer(dst) - reader, err := encryptor.EncryptContents(md5) - if err != nil { - r.Error = err - return - } - - _, err = io.Copy(sha, reader) - if err != nil { - r.Error = err - return - } - - data := encryptor.GetCipherData() - env, err := encodeMeta(md5, data) - if err != nil { - r.Error = err - return - } - - shaHex := hex.EncodeToString(sha.GetValue()) - req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", shaHex) - - dst.Seek(0, sdkio.SeekStart) - input.Body = dst - - err = c.SaveStrategy.Save(env, r) - r.Error = err - }) - - return req, out + return putObjectRequest(c.getClientOptions(), input) } // PutObject is a wrapper for PutObjectRequest +// +// deprecated: See EncryptionClientV2.PutObject func (c *EncryptionClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { - req, out := c.PutObjectRequest(input) - return out, req.Send() + return putObject(c.getClientOptions(), input) } // PutObjectWithContext is a wrapper for PutObjectRequest with the additional @@ -153,9 +94,19 @@ func (c *EncryptionClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOut // Context input parameters. The Context must not be nil. A nil Context will // cause a panic. Use the Context to add deadlining, timeouts, etc. In the future // this may create sub-contexts for individual underlying requests. +// PutObject is a wrapper for PutObjectRequest +// +// deprecated: See EncryptionClientV2.PutObjectWithContext func (c *EncryptionClient) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInput, opts ...request.Option) (*s3.PutObjectOutput, error) { - req, out := c.PutObjectRequest(input) - req.SetContext(ctx) - req.ApplyOptions(opts...) - return out, req.Send() + return putObjectWithContext(c.getClientOptions(), ctx, input, opts...) +} + +func (c *EncryptionClient) getClientOptions() EncryptionClientOptions { + return EncryptionClientOptions{ + S3Client: c.S3Client, + ContentCipherBuilder: c.ContentCipherBuilder, + SaveStrategy: c.SaveStrategy, + TempFolderPath: c.TempFolderPath, + MinFileSize: c.MinFileSize, + } } diff --git a/service/s3/s3crypto/encryption_client_v2.go b/service/s3/s3crypto/encryption_client_v2.go new file mode 100644 index 00000000000..3f8961f2187 --- /dev/null +++ b/service/s3/s3crypto/encryption_client_v2.go @@ -0,0 +1,108 @@ +package s3crypto + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +// EncryptionClientV2 is an S3 crypto client. By default the SDK will use Authentication mode which +// will use KMS for key wrapping and AES GCM for content encryption. +// AES GCM will load all data into memory. However, the rest of the content algorithms +// do not load the entire contents into memory. +type EncryptionClientV2 struct { + options EncryptionClientOptions +} + +// EncryptionClientOptions is the configuration options for EncryptionClientV2 +type EncryptionClientOptions struct { + S3Client s3iface.S3API + ContentCipherBuilder ContentCipherBuilder + // SaveStrategy will dictate where the envelope is saved. + // + // Defaults to the object's metadata + SaveStrategy SaveStrategy + // TempFolderPath is used to store temp files when calling PutObject. + // Temporary files are needed to compute the X-Amz-Content-Sha256 header. + TempFolderPath string + // MinFileSize is the minimum size for the content to write to a + // temporary file instead of using memory. + MinFileSize int64 +} + +// NewEncryptionClientV2 instantiates a new S3 crypto client. An error will be returned to the caller if the provided +// contentCipherBuilder has been deprecated, or if it uses other deprecated components. +// +// Example: +// cmkID := "arn:aws:kms:region:000000000000:key/00000000-0000-0000-0000-000000000000" +// sess := session.Must(session.NewSession()) +// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) +// svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler)) +func NewEncryptionClientV2(prov client.ConfigProvider, contentCipherBuilder ContentCipherBuilder, options ...func(clientOptions *EncryptionClientOptions), +) ( + client *EncryptionClientV2, err error, +) { + s3client := s3.New(prov) + + s3client.Handlers.Build.PushBack(func(r *request.Request) { + request.AddToUserAgent(r, "S3CryptoV2") + }) + + clientOptions := &EncryptionClientOptions{ + S3Client: s3client, + ContentCipherBuilder: contentCipherBuilder, + SaveStrategy: HeaderV2SaveStrategy{}, + MinFileSize: DefaultMinFileSize, + } + + for _, option := range options { + option(clientOptions) + } + + if feature, ok := contentCipherBuilder.(deprecatedFeatures); ok { + if err := feature.isUsingDeprecatedFeatures(); err != nil { + return nil, err + } + } + + client = &EncryptionClientV2{ + *clientOptions, + } + + return client, err +} + +// PutObjectRequest creates a temp file to encrypt the contents into. It then streams +// that data to S3. +// +// Example: +// sess := session.Must(session.NewSession()) +// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), "cmkID") +// svc := s3crypto.NewEncryptionClientV2(sess, s3crypto.AESGCMContentCipherBuilder(handler)) +// req, out := svc.PutObjectRequest(&s3.PutObjectInput { +// Key: aws.String("testKey"), +// Bucket: aws.String("testBucket"), +// Body: strings.NewReader("test data"), +// }) +// err := req.Send() +func (c *EncryptionClientV2) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) { + return putObjectRequest(c.options, input) +} + +// PutObject is a wrapper for PutObjectRequest +func (c *EncryptionClientV2) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + return putObject(c.options, input) +} + +// PutObjectWithContext is a wrapper for PutObjectRequest with the additional +// context, and request options support. +// +// PutObjectWithContext is the same as PutObject with the additional support for +// Context input parameters. The Context must not be nil. A nil Context will +// cause a panic. Use the Context to add deadlining, timeouts, etc. In the future +// this may create sub-contexts for individual underlying requests. +func (c *EncryptionClientV2) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInput, opts ...request.Option) (*s3.PutObjectOutput, error) { + return putObjectWithContext(c.options, ctx, input, opts...) +} diff --git a/service/s3/s3crypto/envelope.go b/service/s3/s3crypto/envelope.go index 6e1e70a4de6..043281010c9 100644 --- a/service/s3/s3crypto/envelope.go +++ b/service/s3/s3crypto/envelope.go @@ -32,6 +32,6 @@ type Envelope struct { WrapAlg string `json:"x-amz-wrap-alg"` CEKAlg string `json:"x-amz-cek-alg"` TagLen string `json:"x-amz-tag-len"` - UnencryptedMD5 string `json:"x-amz-unencrypted-content-md5"` + UnencryptedMD5 string `json:"-"` UnencryptedContentLen string `json:"x-amz-unencrypted-content-length"` } diff --git a/service/s3/s3crypto/integration/main_test.go b/service/s3/s3crypto/integration/main_test.go new file mode 100644 index 00000000000..6fff21828ca --- /dev/null +++ b/service/s3/s3crypto/integration/main_test.go @@ -0,0 +1,187 @@ +// +build integration,go1.14 + +package integration + +import ( + "bytes" + "crypto/rand" + "flag" + "io" + "io/ioutil" + "log" + "os" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/awstesting/integration" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/kms/kmsiface" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3crypto" + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +var config = &struct { + Enabled bool + Region string + KMSKeyID string + Bucket string + Session *session.Session + Clients struct { + KMS kmsiface.KMSAPI + S3 s3iface.S3API + } +}{} + +func init() { + flag.BoolVar(&config.Enabled, "enable", false, "enable integration testing") + flag.StringVar(&config.Region, "region", "us-west-2", "integration test region") + flag.StringVar(&config.KMSKeyID, "kms-key-id", "", "KMS CMK Key ID") + flag.StringVar(&config.Bucket, "bucket", "", "S3 Bucket Name") +} + +func TestMain(m *testing.M) { + flag.Parse() + if !config.Enabled { + log.Println("skipping s3crypto integration tests") + os.Exit(0) + } + + if len(config.Bucket) == 0 { + log.Fatal("bucket name must be provided") + } + + if len(config.KMSKeyID) == 0 { + log.Fatal("kms cmk key id must be provided") + } + + config.Session = session.Must(session.NewSession(&aws.Config{Region: &config.Region})) + + config.Clients.KMS = kms.New(config.Session) + config.Clients.S3 = s3.New(config.Session) + + m.Run() +} + +func TestEncryptionV1_WithV2Interop(t *testing.T) { + kmsKeyGenerator := s3crypto.NewKMSKeyGenerator(config.Clients.KMS, config.KMSKeyID) + + // 1020 is chosen here as it is not cleanly divisible by the AES-256 block size + testData := make([]byte, 1020) + _, err := rand.Read(testData) + if err != nil { + t.Fatalf("failed to read random data: %v", err) + } + + v1DC := s3crypto.NewDecryptionClient(config.Session, func(client *s3crypto.DecryptionClient) { + client.S3Client = config.Clients.S3 + }) + v2DC := s3crypto.NewDecryptionClientV2(config.Session, func(options *s3crypto.DecryptionClientOptions) { + options.S3Client = config.Clients.S3 + }) + + cases := map[string]s3crypto.ContentCipherBuilder{ + "AES/GCM/NoPadding": s3crypto.AESGCMContentCipherBuilder(kmsKeyGenerator), + "AES/CBC/PKCS5Padding": s3crypto.AESCBCContentCipherBuilder(kmsKeyGenerator, s3crypto.AESCBCPadder), + } + + for name, ccb := range cases { + t.Run(name, func(t *testing.T) { + ec := s3crypto.NewEncryptionClient(config.Session, ccb, func(client *s3crypto.EncryptionClient) { + client.S3Client = config.Clients.S3 + }) + id := integration.UniqueID() + // PutObject with V1 Client + putObject(t, ec, id, bytes.NewReader(testData)) + // Verify V1 Decryption Client + getObjectAndCompare(t, v1DC, id, testData) + // Verify V2 Decryption Client + getObjectAndCompare(t, v2DC, id, testData) + }) + } +} + +func TestEncryptionV2(t *testing.T) { + kmsKeyGenerator := s3crypto.NewKMSContextKeyGenerator(config.Clients.KMS, config.KMSKeyID) + gcmContentCipherBuilder := s3crypto.AESGCMContentCipherBuilder(kmsKeyGenerator) + + ec, err := s3crypto.NewEncryptionClientV2(config.Session, gcmContentCipherBuilder, func(options *s3crypto.EncryptionClientOptions) { + options.S3Client = config.Clients.S3 + }) + if err != nil { + t.Fatalf("failed to construct encryption client: %v", err) + } + + dc := s3crypto.NewDecryptionClientV2(config.Session, func(options *s3crypto.DecryptionClientOptions) { + options.S3Client = config.Clients.S3 + }) + + // 1020 is chosen here as it is not cleanly divisible by the AES-256 block size + testData := make([]byte, 1020) + _, err = rand.Read(testData) + if err != nil { + t.Fatalf("failed to read random data: %v", err) + } + + keyId := integration.UniqueID() + + // Upload V2 Objects with Encryption Client + putObject(t, ec, keyId, bytes.NewReader(testData)) + + // Verify V2 Object with Decryption Client + getObjectAndCompare(t, dc, keyId, testData) +} + +type Encryptor interface { + PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) +} + +func putObject(t *testing.T, client Encryptor, key string, reader io.ReadSeeker) { + t.Helper() + _, err := client.PutObject(&s3.PutObjectInput{ + Bucket: &config.Bucket, + Key: &key, + Body: reader, + }) + if err != nil { + t.Fatalf("failed to upload object: %v", err) + } + t.Cleanup(doKeyCleanup(key)) +} + +type Decryptor interface { + GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) +} + +func getObjectAndCompare(t *testing.T, client Decryptor, key string, expected []byte) { + t.Helper() + output, err := client.GetObject(&s3.GetObjectInput{ + Bucket: &config.Bucket, + Key: &key, + }) + if err != nil { + t.Fatalf("failed to get object: %v", err) + } + + actual, err := ioutil.ReadAll(output.Body) + if err != nil { + t.Fatalf("failed to read body response: %v", err) + } + + if bytes.Compare(expected, actual) != 0 { + t.Errorf("expected bytes did not match actual") + } +} + +func doKeyCleanup(key string) func() { + return func() { + _, err := config.Clients.S3.DeleteObject(&s3.DeleteObjectInput{ + Bucket: &config.Bucket, + Key: &key, + }) + if err != nil { + log.Printf("failed to delete %s: %v", key, err) + } + } +} diff --git a/service/s3/s3crypto/key_handler.go b/service/s3/s3crypto/key_handler.go index 6cab2b32400..f74a74d07e4 100644 --- a/service/s3/s3crypto/key_handler.go +++ b/service/s3/s3crypto/key_handler.go @@ -20,6 +20,22 @@ type CipherDataGeneratorWithContext interface { GenerateCipherDataWithContext(aws.Context, int, int) (CipherData, error) } +// CipherDataGeneratorWithCEKAlg handles generating proper key and IVs of proper size for the +// content cipher. CipherDataGenerator will also encrypt the key and store it in +// the CipherData. +type CipherDataGeneratorWithCEKAlg interface { + GenerateCipherDataWithCEKAlg(int, int, string) (CipherData, error) + + CipherDataGenerator // backwards comparability to plug into older interface +} + +// CipherDataGeneratorWithCEKAlgWithContext handles generating proper key and IVs of +// proper size for the content cipher. CipherDataGenerator will also encrypt +// the key and store it in the CipherData. +type CipherDataGeneratorWithCEKAlgWithContext interface { + GenerateCipherDataWithCEKAlgWithContext(aws.Context, int, int, string) (CipherData, error) +} + // CipherDataDecrypter is a handler to decrypt keys from the envelope. type CipherDataDecrypter interface { DecryptKey([]byte) ([]byte, error) @@ -30,8 +46,11 @@ type CipherDataDecrypterWithContext interface { DecryptKeyWithContext(aws.Context, []byte) ([]byte, error) } -func generateBytes(n int) []byte { +func generateBytes(n int) ([]byte, error) { b := make([]byte, n) - rand.Read(b) - return b + _, err := rand.Read(b) + if err != nil { + return nil, err + } + return b, nil } diff --git a/service/s3/s3crypto/key_handler_test.go b/service/s3/s3crypto/key_handler_test.go index 8e0e5ea426a..bd2ba286939 100644 --- a/service/s3/s3crypto/key_handler_test.go +++ b/service/s3/s3crypto/key_handler_test.go @@ -5,15 +5,15 @@ import ( ) func TestGenerateBytes(t *testing.T) { - b := generateBytes(5) + b, _ := generateBytes(5) if e, a := 5, len(b); e != a { t.Errorf("expected %d, but received %d", e, a) } - b = generateBytes(0) + b, _ = generateBytes(0) if e, a := 0, len(b); e != a { t.Errorf("expected %d, but received %d", e, a) } - b = generateBytes(1024) + b, _ = generateBytes(1024) if e, a := 1024, len(b); e != a { t.Errorf("expected %d, but received %d", e, a) } diff --git a/service/s3/s3crypto/kms_key_handler.go b/service/s3/s3crypto/kms_key_handler.go index 91ea010abd0..be1af51929f 100644 --- a/service/s3/s3crypto/kms_key_handler.go +++ b/service/s3/s3crypto/kms_key_handler.go @@ -1,6 +1,8 @@ package s3crypto import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/kms" @@ -10,12 +12,16 @@ import ( const ( // KMSWrap is a constant used during decryption to build a KMS key handler. KMSWrap = "kms" + + // KMSContextWrap is a constant used during decryption to build a kms+context key handler + KMSContextWrap = "kms+context" ) // kmsKeyHandler will make calls to KMS to get the masterkey type kmsKeyHandler struct { - kms kmsiface.KMSAPI - cmkID *string + kms kmsiface.KMSAPI + cmkID *string + withContext bool CipherData } @@ -28,35 +34,73 @@ type kmsKeyHandler struct { // cmkID := "arn to key" // matdesc := s3crypto.MaterialDescription{} // handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID) +// +// deprecated: See NewKMSContextKeyGenerator func NewKMSKeyGenerator(kmsClient kmsiface.KMSAPI, cmkID string) CipherDataGenerator { return NewKMSKeyGeneratorWithMatDesc(kmsClient, cmkID, MaterialDescription{}) } -// NewKMSKeyGeneratorWithMatDesc builds a new KMS key provider using the customer key ID and material +// NewKMSContextKeyGenerator builds a new kms+context key provider using the customer key ID and material // description. // // Example: // sess := session.New(&aws.Config{}) // cmkID := "arn to key" // matdesc := s3crypto.MaterialDescription{} -// handler := s3crypto.NewKMSKeyGeneratorWithMatDesc(kms.New(sess), cmkID, matdesc) -func NewKMSKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGenerator { +// handler := s3crypto.NewKMSContextKeyGenerator(kms.New(sess), cmkID) +func NewKMSContextKeyGenerator(client kmsiface.KMSAPI, cmkID string) CipherDataGeneratorWithCEKAlg { + return NewKMSContextKeyGeneratorWithMatDesc(client, cmkID, MaterialDescription{}) +} + +func newKMSKeyHandler(client kmsiface.KMSAPI, cmkID string, withContext bool, matdesc MaterialDescription) *kmsKeyHandler { + // These values are read only making them thread safe + kp := &kmsKeyHandler{ + kms: client, + cmkID: &cmkID, + withContext: withContext, + } + if matdesc == nil { matdesc = MaterialDescription{} } - matdesc["kms_cmk_id"] = &cmkID // These values are read only making them thread safe - kp := &kmsKeyHandler{ - kms: kmsClient, - cmkID: &cmkID, + if kp.withContext { + kp.CipherData.WrapAlgorithm = KMSContextWrap + } else { + matdesc["kms_cmk_id"] = &cmkID + kp.CipherData.WrapAlgorithm = KMSWrap } - // These values are read only making them thread safe - kp.CipherData.WrapAlgorithm = KMSWrap kp.CipherData.MaterialDescription = matdesc return kp } +// NewKMSKeyGeneratorWithMatDesc builds a new KMS key provider using the customer key ID and material +// description. +// +// Example: +// sess := session.New(&aws.Config{}) +// cmkID := "arn to key" +// matdesc := s3crypto.MaterialDescription{} +// handler := s3crypto.NewKMSKeyGeneratorWithMatDesc(kms.New(sess), cmkID, matdesc) +// +// deprecated: See NewKMSContextKeyGeneratorWithMatDesc +func NewKMSKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGenerator { + return newKMSKeyHandler(kmsClient, cmkID, false, matdesc) +} + +// NewKMSContextKeyGeneratorWithMatDesc builds a new kms+context key provider using the customer key ID and material +// description. +// +// Example: +// sess := session.New(&aws.Config{}) +// cmkID := "arn to key" +// matdesc := s3crypto.MaterialDescription{} +// handler := s3crypto.NewKMSKeyGeneratorWithMatDesc(kms.New(sess), cmkID, matdesc) +func NewKMSContextKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGeneratorWithCEKAlg { + return newKMSKeyHandler(kmsClient, cmkID, true, matdesc) +} + // NewKMSWrapEntry builds returns a new KMS key provider and its decrypt handler. // // Example: @@ -67,6 +111,8 @@ func NewKMSKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matd // svc := s3crypto.NewDecryptionClient(sess, func(svc *s3crypto.DecryptionClient) { // svc.WrapRegistry[s3crypto.KMSWrap] = decryptHandler // })) +// +// deprecated: See NewKMSContextWrapEntry func NewKMSWrapEntry(kmsClient kmsiface.KMSAPI) WrapEntry { // These values are read only making them thread safe kp := &kmsKeyHandler{ @@ -76,6 +122,26 @@ func NewKMSWrapEntry(kmsClient kmsiface.KMSAPI) WrapEntry { return kp.decryptHandler } +// NewKMSContextWrapEntry builds returns a new KMS key provider and its decrypt handler. +// +// Example: +// sess := session.New(&aws.Config{}) +// customKMSClient := kms.New(sess) +// decryptHandler := s3crypto.NewKMSContextWrapEntry(customKMSClient) +// +// svc := s3crypto.NewDecryptionClient(sess, func(svc *s3crypto.DecryptionClient) { +// svc.WrapRegistry[s3crypto.KMSContextWrap] = decryptHandler +// })) +func NewKMSContextWrapEntry(kmsClient kmsiface.KMSAPI) WrapEntry { + // These values are read only making them thread safe + kp := &kmsKeyHandler{ + kms: kmsClient, + withContext: true, + } + + return kp.decryptHandler +} + // decryptHandler initializes a KMS keyprovider with a material description. This // is used with Decrypting kms content, due to the cmkID being in the material description. func (kp kmsKeyHandler) decryptHandler(env Envelope) (CipherDataDecrypter, error) { @@ -86,13 +152,16 @@ func (kp kmsKeyHandler) decryptHandler(env Envelope) (CipherDataDecrypter, error } cmkID, ok := m["kms_cmk_id"] - if !ok { + if !kp.withContext && !ok { return nil, awserr.New("MissingCMKIDError", "Material description is missing CMK ID", nil) } kp.CipherData.MaterialDescription = m kp.cmkID = cmkID kp.WrapAlgorithm = KMSWrap + if kp.withContext { + kp.WrapAlgorithm = KMSContextWrap + } return &kp, nil } @@ -121,12 +190,31 @@ func (kp *kmsKeyHandler) GenerateCipherData(keySize, ivSize int) (CipherData, er return kp.GenerateCipherDataWithContext(aws.BackgroundContext(), keySize, ivSize) } +func (kp kmsKeyHandler) GenerateCipherDataWithCEKAlg(keySize, ivSize int, cekAlgorithm string) (CipherData, error) { + return kp.GenerateCipherDataWithCEKAlgWithContext(aws.BackgroundContext(), keySize, ivSize, cekAlgorithm) +} + // GenerateCipherDataWithContext makes a call to KMS to generate a data key, // Upon making the call, it also sets the encrypted key. func (kp *kmsKeyHandler) GenerateCipherDataWithContext(ctx aws.Context, keySize, ivSize int) (CipherData, error) { + return kp.GenerateCipherDataWithCEKAlgWithContext(ctx, keySize, ivSize, "") +} + +func (kp kmsKeyHandler) GenerateCipherDataWithCEKAlgWithContext(ctx aws.Context, keySize int, ivSize int, cekAlgorithm string) (CipherData, error) { + md := kp.CipherData.MaterialDescription + + wrapAlgorithm := KMSWrap + if kp.withContext { + wrapAlgorithm = KMSContextWrap + if len(cekAlgorithm) == 0 { + return CipherData{}, fmt.Errorf("CEK algorithm identifier must not be empty") + } + md["aws:"+cekAlgorithmHeader] = &cekAlgorithm + } + out, err := kp.kms.GenerateDataKeyWithContext(ctx, &kms.GenerateDataKeyInput{ - EncryptionContext: kp.CipherData.MaterialDescription, + EncryptionContext: md, KeyId: kp.cmkID, KeySpec: aws.String("AES_256"), }) @@ -134,13 +222,24 @@ func (kp *kmsKeyHandler) GenerateCipherDataWithContext(ctx aws.Context, keySize, return CipherData{}, err } - iv := generateBytes(ivSize) + iv, err := generateBytes(ivSize) + if err != nil { + return CipherData{}, err + } + cd := CipherData{ Key: out.Plaintext, IV: iv, - WrapAlgorithm: KMSWrap, - MaterialDescription: kp.CipherData.MaterialDescription, + WrapAlgorithm: wrapAlgorithm, + MaterialDescription: md, EncryptedKey: out.CiphertextBlob, } return cd, nil } + +func (kp *kmsKeyHandler) isUsingDeprecatedFeatures() error { + if !kp.withContext { + return errDeprecatedCipherDataGenerator + } + return nil +} diff --git a/service/s3/s3crypto/kms_key_handler_test.go b/service/s3/s3crypto/kms_key_handler_test.go index b12d76d97d4..4c030b0e9cf 100644 --- a/service/s3/s3crypto/kms_key_handler_test.go +++ b/service/s3/s3crypto/kms_key_handler_test.go @@ -4,7 +4,9 @@ import ( "bytes" "encoding/base64" "encoding/hex" + "encoding/json" "fmt" + "io/ioutil" "net/http" "net/http/httptest" "reflect" @@ -105,10 +107,98 @@ func TestKMSDecrypt(t *testing.T) { } } -func TestKMSDecryptBadJSON(t *testing.T) { +func TestKMSContextGenerateCipherData(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + return + } + var body map[string]interface{} + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + w.WriteHeader(500) + return + } + + md, ok := body["EncryptionContext"].(map[string]interface{}) + if !ok { + w.WriteHeader(500) + return + } + + exEncContext := map[string]interface{}{ + "aws:" + cekAlgorithmHeader: "cekAlgValue", + } + + if e, a := exEncContext, md; !reflect.DeepEqual(e, a) { + w.WriteHeader(500) + t.Errorf("expected %v, got %v", e, a) + return + } + + fmt.Fprintln(w, `{"CiphertextBlob":"AQEDAHhqBCCY1MSimw8gOGcUma79cn4ANvTtQyv9iuBdbcEF1QAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ6IcN5E4wVbk38MNAIBEIA7oF1E3lS7FY9DkoxPc/UmJsEwHzL82zMqoLwXIvi8LQHr8If4Lv6zKqY8u0+JRgSVoqCvZDx3p8Cn6nM=","KeyId":"arn:aws:kms:us-west-2:042062605278:key/c80a5cdb-8d09-4f9f-89ee-df01b2e3870a","Plaintext":"6tmyz9JLBE2yIuU7iXpArqpDVle172WSmxjcO6GNT7E="}`) + })) + defer ts.Close() + + sess := unit.Session.Copy(&aws.Config{ + MaxRetries: aws.Int(0), + Endpoint: aws.String(ts.URL), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + Region: aws.String("us-west-2"), + }) + + svc := kms.New(sess) + handler := NewKMSContextKeyGenerator(svc, "testid") + + keySize := 32 + ivSize := 16 + + cd, err := handler.GenerateCipherDataWithCEKAlg(keySize, ivSize, "cekAlgValue") + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + if keySize != len(cd.Key) { + t.Errorf("expected %d, but received %d", keySize, len(cd.Key)) + } + if ivSize != len(cd.IV) { + t.Errorf("expected %d, but received %d", ivSize, len(cd.IV)) + } +} + +func TestKMSContextDecrypt(t *testing.T) { key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22") keyB64 := base64.URLEncoding.EncodeToString(key) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + return + } + var body map[string]interface{} + err = json.Unmarshal(bodyBytes, &body) + if err != nil { + w.WriteHeader(500) + return + } + + md, ok := body["EncryptionContext"].(map[string]interface{}) + if !ok { + w.WriteHeader(500) + return + } + + exEncContext := map[string]interface{}{ + "aws:" + cekAlgorithmHeader: "cekAlgValue", + } + + if e, a := exEncContext, md; !reflect.DeepEqual(e, a) { + w.WriteHeader(500) + t.Errorf("expected %v, got %v", e, a) + return + } + fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`)) })) defer ts.Close() @@ -120,9 +210,17 @@ func TestKMSDecryptBadJSON(t *testing.T) { S3ForcePathStyle: aws.Bool(true), Region: aws.String("us-west-2"), }) + handler, err := NewKMSContextWrapEntry(kms.New(sess))(Envelope{MatDesc: `{"aws:x-amz-cek-alg": "cekAlgValue"}`}) + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + + plaintextKey, err := handler.DecryptKey([]byte{1, 2, 3, 4}) + if err != nil { + t.Errorf("expected no error, but received %v", err) + } - _, err := (kmsKeyHandler{kms: kms.New(sess)}).decryptHandler(Envelope{MatDesc: `{"kms_cmk_id":"test"`}) - if err == nil { - t.Errorf("expected error, but received none") + if !bytes.Equal(key, plaintextKey) { + t.Errorf("expected %v, but received %v", key, plaintextKey) } } diff --git a/service/s3/s3crypto/migrations_test.go b/service/s3/s3crypto/migrations_test.go new file mode 100644 index 00000000000..6d76809d3cb --- /dev/null +++ b/service/s3/s3crypto/migrations_test.go @@ -0,0 +1,148 @@ +package s3crypto_test + +import ( + "bytes" + "fmt" + "io/ioutil" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3crypto" +) + +// ExampleNewEncryptionClientV2_migration00 provides a migration example for how users can migrate from the V1 +// encryption client to the V2 encryption client. This example demonstrates how an application using the `kms` key wrap +// algorithm with `AES/CBC/PKCS5Padding` can migrate their application to `kms+context` key wrapping with +// `AES/GCM/NoPadding` content encryption. +func ExampleNewEncryptionClientV2_migration00() { + sess := session.Must(session.NewSession()) + kmsClient := kms.New(sess) + cmkID := "1234abcd-12ab-34cd-56ef-1234567890ab" + + // Usage of NewKMSKeyGenerator (kms) key wrapping algorithm must be migrated to NewKMSContextKeyGenerator (kms+context) key wrapping algorithm + // + // cipherDataGenerator := s3crypto.NewKMSKeyGenerator(kmsClient, cmkID) + cipherDataGenerator := s3crypto.NewKMSContextKeyGenerator(kmsClient, cmkID) + + // Usage of AESCBCContentCipherBuilder (AES/CBC/PKCS5Padding) must be migrated to AESGCMContentCipherBuilder (AES/GCM/NoPadding) + // + // contentCipherBuilder := s3crypto.AESCBCContentCipherBuilder(cipherDataGenerator, s3crypto.AESCBCPadder) + contentCipherBuilder := s3crypto.AESGCMContentCipherBuilder(cipherDataGenerator) + + // Construction of an encryption client should be done using NewEncryptionClientV2 + // + // encryptionClient := s3crypto.NewEncryptionClient(sess, contentCipherBuilder) + encryptionClient, err := s3crypto.NewEncryptionClientV2(sess, contentCipherBuilder) + if err != nil { + fmt.Printf("failed to construct encryption client: %v", err) + return + } + + _, err = encryptionClient.PutObject(&s3.PutObjectInput{ + Bucket: aws.String("your_bucket"), + Key: aws.String("your_key"), + Body: bytes.NewReader([]byte("your content")), + }) + if err != nil { + fmt.Printf("put object error: %v\n", err) + return + } + fmt.Println("put object completed") +} + +// ExampleNewEncryptionClientV2_migration01 provides a more advanced migration example for how users can +// migrate from the V1 encryption client to the V2 encryption client using more complex client construction. +func ExampleNewEncryptionClientV2_migration01() { + sess := session.Must(session.NewSession()) + kmsClient := kms.New(sess) + cmkID := "1234abcd-12ab-34cd-56ef-1234567890ab" + + cipherDataGenerator := s3crypto.NewKMSContextKeyGenerator(kmsClient, cmkID) + + contentCipherBuilder := s3crypto.AESGCMContentCipherBuilder(cipherDataGenerator) + + // Overriding of the encryption client options is possible by passing in functional arguments that override the + // provided EncryptionClientOptions. + // + // encryptionClient := s3crypto.NewEncryptionClient(cipherDataGenerator, contentCipherBuilder, func(o *s3crypto.EncryptionClient) { + // o.S3Client = s3.New(sess, &aws.Config{Region: aws.String("us-west-2")}), + // }) + encryptionClient, err := s3crypto.NewEncryptionClientV2(sess, contentCipherBuilder, func(o *s3crypto.EncryptionClientOptions) { + o.S3Client = s3.New(sess, &aws.Config{Region: aws.String("us-west-2")}) + }) + if err != nil { + fmt.Printf("failed to construct encryption client: %v", err) + return + } + + _, err = encryptionClient.PutObject(&s3.PutObjectInput{ + Bucket: aws.String("your_bucket"), + Key: aws.String("your_key"), + Body: bytes.NewReader([]byte("your content")), + }) + if err != nil { + fmt.Printf("put object error: %v\n", err) + return + } + fmt.Println("put object completed") +} + +// ExampleNewDecryptionClientV2_migration00 provides a migration example for how users can migrate +// from the V1 Decryption Clients to the V2 Decryption Clients. +func ExampleNewDecryptionClientV2_migration00() { + sess := session.Must(session.NewSession()) + + // Construction of an decryption client must be done using NewDecryptionClientV2 + // The V2 decryption client is able to decrypt object encrypted by the V1 client. + // + // decryptionClient := s3crypto.NewDecryptionClient(sess) + decryptionClient := s3crypto.NewDecryptionClientV2(sess) + + getObject, err := decryptionClient.GetObject(&s3.GetObjectInput{ + Bucket: aws.String("your_bucket"), + Key: aws.String("your_key"), + }) + if err != nil { + fmt.Printf("get object error: %v\n", err) + return + } + + _, err = ioutil.ReadAll(getObject.Body) + if err != nil { + fmt.Printf("error reading object: %v\n", err) + } + fmt.Println("get object completed") +} + +// ExampleNewDecryptionClientV2_migration01 provides a more advanced migration example for how users can +// migrate from the V1 decryption client to the V2 decryption client using more complex client construction. +func ExampleNewDecryptionClientV2_migration01() { + sess := session.Must(session.NewSession()) + + // Construction of an decryption client must be done using NewDecryptionClientV2 + // The V2 decryption client is able to decrypt object encrypted by the V1 client. + // + // decryptionClient := s3crypto.NewDecryptionClient(sess, func(o *s3crypto.DecryptionClient) { + // o.S3Client = s3.New(sess, &aws.Config{Region: aws.String("us-west-2")}) + //}) + decryptionClient := s3crypto.NewDecryptionClientV2(sess, func(o *s3crypto.DecryptionClientOptions) { + o.S3Client = s3.New(sess, &aws.Config{Region: aws.String("us-west-2")}) + }) + + getObject, err := decryptionClient.GetObject(&s3.GetObjectInput{ + Bucket: aws.String("your_bucket"), + Key: aws.String("your_key"), + }) + if err != nil { + fmt.Printf("get object error: %v\n", err) + return + } + + _, err = ioutil.ReadAll(getObject.Body) + if err != nil { + fmt.Printf("error reading object: %v\n", err) + } + fmt.Println("get object completed") +} diff --git a/service/s3/s3crypto/mock_test.go b/service/s3/s3crypto/mock_test.go index 4ad309c45b6..ac96d4cbf37 100644 --- a/service/s3/s3crypto/mock_test.go +++ b/service/s3/s3crypto/mock_test.go @@ -8,8 +8,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3crypto" ) -type mockGenerator struct { -} +type mockGenerator struct{} func (m mockGenerator) GenerateCipherData(keySize, ivSize int) (s3crypto.CipherData, error) { cd := s3crypto.CipherData{ @@ -27,7 +26,6 @@ func (m mockGenerator) EncryptKey(key []byte) ([]byte, error) { func (m mockGenerator) DecryptKey(key []byte) ([]byte, error) { return make([]byte, 16), nil - } type mockCipherBuilder struct { diff --git a/service/s3/s3crypto/shared_client.go b/service/s3/s3crypto/shared_client.go new file mode 100644 index 00000000000..2c967d07467 --- /dev/null +++ b/service/s3/s3crypto/shared_client.go @@ -0,0 +1,210 @@ +package s3crypto + +import ( + "encoding/base64" + "encoding/hex" + "io" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/internal/sdkio" + "github.com/aws/aws-sdk-go/service/s3" +) + +func putObjectRequest(c EncryptionClientOptions, input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) { + req, out := c.S3Client.PutObjectRequest(input) + + // Get Size of file + n, err := aws.SeekerLen(input.Body) + if err != nil { + req.Error = err + return req, out + } + + dst, err := getWriterStore(req, c.TempFolderPath, n >= c.MinFileSize) + if err != nil { + req.Error = err + return req, out + } + + req.Handlers.Build.PushFront(func(r *request.Request) { + if err != nil { + r.Error = err + return + } + var encryptor ContentCipher + if v, ok := c.ContentCipherBuilder.(ContentCipherBuilderWithContext); ok { + encryptor, err = v.ContentCipherWithContext(r.Context()) + } else { + encryptor, err = c.ContentCipherBuilder.ContentCipher() + } + if err != nil { + r.Error = err + return + } + + md5 := newMD5Reader(input.Body) + sha := newSHA256Writer(dst) + reader, err := encryptor.EncryptContents(md5) + if err != nil { + r.Error = err + return + } + + _, err = io.Copy(sha, reader) + if err != nil { + r.Error = err + return + } + + data := encryptor.GetCipherData() + env, err := encodeMeta(md5, data) + if err != nil { + r.Error = err + return + } + + shaHex := hex.EncodeToString(sha.GetValue()) + req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", shaHex) + + dst.Seek(0, sdkio.SeekStart) + input.Body = dst + + err = c.SaveStrategy.Save(env, r) + r.Error = err + }) + + return req, out +} + +func putObject(options EncryptionClientOptions, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + req, out := putObjectRequest(options, input) + return out, req.Send() +} + +func putObjectWithContext(options EncryptionClientOptions, ctx aws.Context, input *s3.PutObjectInput, opts ...request.Option) (*s3.PutObjectOutput, error) { + req, out := putObjectRequest(options, input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +func getObjectRequest(options DecryptionClientOptions, input *s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput) { + req, out := options.S3Client.GetObjectRequest(input) + req.Handlers.Unmarshal.PushBack(func(r *request.Request) { + env, err := options.LoadStrategy.Load(r) + if err != nil { + r.Error = err + out.Body.Close() + return + } + + // If KMS should return the correct CEK algorithm with the proper + // KMS key provider + cipher, err := contentCipherFromEnvelope(options, r.Context(), env) + if err != nil { + r.Error = err + out.Body.Close() + return + } + + reader, err := cipher.DecryptContents(out.Body) + if err != nil { + r.Error = err + out.Body.Close() + return + } + out.Body = reader + }) + return req, out +} + +func getObject(options DecryptionClientOptions, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + req, out := getObjectRequest(options, input) + return out, req.Send() +} + +func getObjectWithContext(options DecryptionClientOptions, ctx aws.Context, input *s3.GetObjectInput, opts ...request.Option) (*s3.GetObjectOutput, error) { + req, out := getObjectRequest(options, input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +func contentCipherFromEnvelope(options DecryptionClientOptions, ctx aws.Context, env Envelope) (ContentCipher, error) { + wrap, err := wrapFromEnvelope(options, env) + if err != nil { + return nil, err + } + + return cekFromEnvelope(options, ctx, env, wrap) +} + +func wrapFromEnvelope(options DecryptionClientOptions, env Envelope) (CipherDataDecrypter, error) { + f, ok := options.WrapRegistry[env.WrapAlg] + if !ok || f == nil { + return nil, awserr.New( + "InvalidWrapAlgorithmError", + "wrap algorithm isn't supported, "+env.WrapAlg, + nil, + ) + } + return f(env) +} + +func cekFromEnvelope(options DecryptionClientOptions, ctx aws.Context, env Envelope, decrypter CipherDataDecrypter) (ContentCipher, error) { + f, ok := options.CEKRegistry[env.CEKAlg] + if !ok || f == nil { + return nil, awserr.New( + "InvalidCEKAlgorithmError", + "cek algorithm isn't supported, "+env.CEKAlg, + nil, + ) + } + + key, err := base64.StdEncoding.DecodeString(env.CipherKey) + if err != nil { + return nil, err + } + + iv, err := base64.StdEncoding.DecodeString(env.IV) + if err != nil { + return nil, err + } + + if d, ok := decrypter.(CipherDataDecrypterWithContext); ok { + key, err = d.DecryptKeyWithContext(ctx, key) + } else { + key, err = decrypter.DecryptKey(key) + } + + if err != nil { + return nil, err + } + + cd := CipherData{ + Key: key, + IV: iv, + CEKAlgorithm: env.CEKAlg, + Padder: getPadder(options, env.CEKAlg), + } + return f(cd) +} + +// getPadder will return an unpadder with checking the cek algorithm specific padder. +// If there wasn't a cek algorithm specific padder, we check the padder itself. +// We return a no unpadder, if no unpadder was found. This means any customization +// either contained padding within the cipher implementation, and to maintain +// backwards compatility we will simply not unpad anything. +func getPadder(options DecryptionClientOptions, cekAlg string) Padder { + padder, ok := options.PadderRegistry[cekAlg] + if !ok { + padder, ok = options.PadderRegistry[cekAlg[strings.LastIndex(cekAlg, "/")+1:]] + if !ok { + return NoPadder + } + } + return padder +} diff --git a/service/s3/s3crypto/strategy.go b/service/s3/s3crypto/strategy.go index 9c180eabd6b..390b151a6bd 100644 --- a/service/s3/s3crypto/strategy.go +++ b/service/s3/s3crypto/strategy.go @@ -63,7 +63,6 @@ func (strat HeaderV2SaveStrategy) Save(env Envelope, req *request.Request) error input.Metadata[http.CanonicalHeaderKey(matDescHeader)] = &env.MatDesc input.Metadata[http.CanonicalHeaderKey(wrapAlgorithmHeader)] = &env.WrapAlg input.Metadata[http.CanonicalHeaderKey(cekAlgorithmHeader)] = &env.CEKAlg - input.Metadata[http.CanonicalHeaderKey(unencryptedMD5Header)] = &env.UnencryptedMD5 input.Metadata[http.CanonicalHeaderKey(unencryptedContentLengthHeader)] = &env.UnencryptedContentLen if len(env.TagLen) > 0 { diff --git a/service/s3/s3crypto/strategy_test.go b/service/s3/s3crypto/strategy_test.go index 10e3d864b19..1c325e3cef4 100644 --- a/service/s3/s3crypto/strategy_test.go +++ b/service/s3/s3crypto/strategy_test.go @@ -1,11 +1,16 @@ package s3crypto_test import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" "reflect" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/awstesting/unit" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3crypto" ) @@ -33,7 +38,6 @@ func TestHeaderV2SaveStrategy(t *testing.T) { "X-Amz-Wrap-Alg": aws.String(s3crypto.KMSWrap), "X-Amz-Cek-Alg": aws.String(s3crypto.AESGCMNoPadding), "X-Amz-Tag-Len": aws.String("128"), - "X-Amz-Unencrypted-Content-Md5": aws.String("hello"), "X-Amz-Unencrypted-Content-Length": aws.String("0"), }, }, @@ -53,7 +57,6 @@ func TestHeaderV2SaveStrategy(t *testing.T) { "X-Amz-Matdesc": aws.String("{}"), "X-Amz-Wrap-Alg": aws.String(s3crypto.KMSWrap), "X-Amz-Cek-Alg": aws.String(s3crypto.AESGCMNoPadding), - "X-Amz-Unencrypted-Content-Md5": aws.String("hello"), "X-Amz-Unencrypted-Content-Length": aws.String("0"), }, }, @@ -75,3 +78,103 @@ func TestHeaderV2SaveStrategy(t *testing.T) { } } } + +func TestS3SaveStrategy(t *testing.T) { + cases := []struct { + env s3crypto.Envelope + expected s3crypto.Envelope + }{ + { + s3crypto.Envelope{ + CipherKey: "Foo", + IV: "Bar", + MatDesc: "{}", + WrapAlg: s3crypto.KMSWrap, + CEKAlg: s3crypto.AESGCMNoPadding, + TagLen: "128", + UnencryptedMD5: "hello", + UnencryptedContentLen: "0", + }, + s3crypto.Envelope{ + CipherKey: "Foo", + IV: "Bar", + MatDesc: "{}", + WrapAlg: s3crypto.KMSWrap, + CEKAlg: s3crypto.AESGCMNoPadding, + TagLen: "128", + UnencryptedContentLen: "0", + }, + }, + { + s3crypto.Envelope{ + CipherKey: "Foo", + IV: "Bar", + MatDesc: "{}", + WrapAlg: s3crypto.KMSWrap, + CEKAlg: s3crypto.AESGCMNoPadding, + UnencryptedMD5: "hello", + UnencryptedContentLen: "0", + }, + s3crypto.Envelope{ + CipherKey: "Foo", + IV: "Bar", + MatDesc: "{}", + WrapAlg: s3crypto.KMSWrap, + CEKAlg: s3crypto.AESGCMNoPadding, + UnencryptedContentLen: "0", + }, + }, + } + + for _, c := range cases { + params := &s3.PutObjectInput{ + Bucket: aws.String("fooBucket"), + Key: aws.String("barKey"), + } + req := &request.Request{ + Params: params, + } + + client := s3.New(unit.Session) + + client.Handlers.Send.Clear() + client.Handlers.Unmarshal.Clear() + client.Handlers.UnmarshalMeta.Clear() + client.Handlers.UnmarshalError.Clear() + client.Handlers.Send.PushBack(func(r *request.Request) { + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + r.HTTPResponse = &http.Response{ + StatusCode: 500, + Body: ioutil.NopCloser(bytes.NewReader([]byte(err.Error()))), + } + return + } + + var actual s3crypto.Envelope + err = json.Unmarshal(bodyBytes, &actual) + if err != nil { + r.HTTPResponse = &http.Response{ + StatusCode: 500, + Body: ioutil.NopCloser(bytes.NewReader([]byte(err.Error()))), + } + return + } + + if e, a := c.expected, actual; !reflect.DeepEqual(e, a) { + t.Errorf("expected %v, got %v", e, a) + } + + r.HTTPResponse = &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + }) + + strat := s3crypto.S3SaveStrategy{Client: client} + err := strat.Save(c.env, req) + if err != nil { + t.Errorf("expected no error, but received %v", err) + } + } +} diff --git a/service/s3/s3crypto/testdata/aes_gcm.json b/service/s3/s3crypto/testdata/aes_gcm.json new file mode 100644 index 00000000000..b7065eb20c3 --- /dev/null +++ b/service/s3/s3crypto/testdata/aes_gcm.json @@ -0,0 +1,56 @@ +[ + { + "comment": "AES-128-GCM", + "key": "DA2FDB0CED551AEB723D8AC1A267CEF3", + "pt": "", + "aad": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "iv": "A5F5160B7B0B025757ACCDAA", + "ct": "", + "tag": "7AD0758C4FA9B8660AA0687B3E7BD517" + }, + { + "comment": "AES-128-GCM", + "key": "4194935CF4524DF93D62FEDBC818D8AC", + "pt": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "aad": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "iv": "0C5A8F5AF7F6064C0130EE64", + "ct": "3F4CC9A7451717E5E939D294A1362B32C274D06411188DAD76AEE3EE4DA46483EA4C1AF38B9B74D7AD2FD8E310CF82", + "tag": "AD563FD10E1EFA3F26753F46E09DB3A0" + }, + { + "comment": "AES-128-GCM", + "key": "AD03EE2FD6048DB7158CEC55D3D760BC", + "pt": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "aad": "", + "iv": "1B813A16DDCB7F08D26E2541", + "ct": "ADD161BE957AE9EC3CEE6600C77FF81D64A80242A510A9D5AD872096C79073B61E8237FAA7D63A3301EA58EC11332C", + "tag": "01944370EC28601ADC989DE05A794AEB" + }, + { + "comment": "AES-256-GCM", + "key": "20142E898CD2FD980FBF34DE6BC85C14DA7D57BD28F4AA5CF1728AB64E843142", + "pt": "", + "aad": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "iv": "FB7B4A824E82DAA6C8BC1251", + "ct": "", + "tag": "81C0E42BB195E262CB3B3A74A0DAE1C8" + }, + { + "comment": "AES-256-GCM", + "key": "D211F278A44EAB666B1021F4B4F60BA6B74464FA9CB7B134934D7891E1479169", + "pt": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "aad": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "iv": "6B5CD3705A733C1AD943D58A", + "ct": "4C25ABD66D3A1BCCE794ACAAF4CEFDF6D2552F4A82C50A98CB15B4812FF557ABE564A9CEFF15F32DCF5A5AA7894888", + "tag": "03EDE71EC952E65AE7B4B85CFEC7D304" + }, + { + "comment": "AES-256-GCM", + "key": "CFE8BFE61B89AF53D2BECE744D27B78C9E4D74D028CE88ED10A422285B1201C9", + "pt": "167B5C226177733A782D616D7A2D63656B2D616C675C223A205C224145532F47434D2F4E6F50616464696E675C227D", + "aad": "", + "iv": "5F08EFBFB7BF5BA365D9EB1D", + "ct": "0A7E82F1E5C76C69679671EEAEE455936F2C4FCCD9DDF1FAA27075E2040644938920C5D16C69E4D93375487B9A80D4", + "tag": "04347D0C5B0E0DE89E033D04D0493DCA" + } +]