security: Encrypt region boundary keys, Part 1 - helper functions

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9
 	github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de
 	github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d
-	github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b
+	github.com/pingcap/kvproto v0.0.0-20200916031750-f9473f2c5379
 	github.com/pingcap/log v0.0.0-20200511115504-543df19646ad
 	github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a
 	github.com/prometheus/client_golang v1.0.0

go.sum

@@ -300,8 +300,8 @@ github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+
 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
 github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b h1:WFBUlCITLWmU7h2d1NS29s2xvbiF/+o37OcW92iIRac=
-github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
+github.com/pingcap/kvproto v0.0.0-20200916031750-f9473f2c5379 h1:KAGE4PYxYLL/dnui3sRCcQHNpcpP5aMl0R/NKzATGgI=
+github.com/pingcap/kvproto v0.0.0-20200916031750-f9473f2c5379/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=

pkg/encryption/crypter.go

@@ -0,0 +1,205 @@
+// Copyright 2020 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryption
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/binary"
+	"io"
+	"time"
+
+	"github.com/pingcap/kvproto/pkg/encryptionpb"
+	"github.com/tikv/pd/pkg/errs"
+)
+
+const (
+	ivLengthCTR = 16
+	ivLengthGCM = 12
+)
+
+// CheckEncryptionMethodSupported check whether the encryption method is currently supported.
+// This is to handle future extension to encryption methods on kvproto side.
+func CheckEncryptionMethodSupported(method encryptionpb.EncryptionMethod) error {
+	switch method {
+	case encryptionpb.EncryptionMethod_AES128_CTR:
+		return nil
+	case encryptionpb.EncryptionMethod_AES192_CTR:
+		return nil
+	case encryptionpb.EncryptionMethod_AES256_CTR:
+		return nil
+	default:
+		name, ok := encryptionpb.EncryptionMethod_name[int32(method)]
+		if ok {
+			return errs.ErrEncryptionInvalidMethod.GenWithStackByArgs(name)
+		}
+		return errs.ErrEncryptionInvalidMethod.GenWithStackByArgs(int32(method))
+	}
+}
+
+// KeyLength return the encryption key length for supported encryption methods.
+func KeyLength(method encryptionpb.EncryptionMethod) (int, error) {
+	switch method {
+	case encryptionpb.EncryptionMethod_AES128_CTR:
+		return 16, nil
+	case encryptionpb.EncryptionMethod_AES192_CTR:
+		return 24, nil
+	case encryptionpb.EncryptionMethod_AES256_CTR:
+		return 32, nil
+	default:
+		name, ok := encryptionpb.EncryptionMethod_name[int32(method)]
+		if ok {
+			return 0, errs.ErrEncryptionInvalidMethod.GenWithStackByArgs(name)
+		}
+		return 0, errs.ErrEncryptionInvalidMethod.GenWithStackByArgs(int32(method))
+	}
+}
+
+// IvCtr represent IV bytes for CTR mode.
+type IvCtr []byte
+
+// IvGcm represent IV bytes for GCM mode.
+type IvGcm []byte
+
+func newIV(ivLength int) ([]byte, error) {
+	iv := make([]byte, ivLength)
+	n, err := io.ReadFull(rand.Reader, iv)
+	if err != nil {
+		return nil, errs.ErrEncryptionGenerateIV.Wrap(err).GenWithStackByArgs()
+	}
+	if n != ivLength {
+		return nil, errs.ErrEncryptionGenerateIV.GenWithStack(
+			"iv length exepcted %d vs actual %d", ivLength, n)
+	}
+	return iv, nil
+}
+
+// NewIvCtr randomly generate an IV for CTR mode.
+func NewIvCtr() (IvCtr, error) {
+	return newIV(ivLengthCTR)
+}
+
+// NewIvGcm randomly generate an IV for GCM mode.
+func NewIvGcm() (IvGcm, error) {
+	return newIV(ivLengthGCM)
+}
+
+// NewDataKey randomly generate a new data key.
+func NewDataKey(
+	method encryptionpb.EncryptionMethod,
+) (keyID uint64, key *encryptionpb.DataKey, err error) {
+	err = CheckEncryptionMethodSupported(method)
+	if err != nil {
+		return
+	}
+	keyIDBuf := make([]byte, 8)
+	n, err := io.ReadFull(rand.Reader, keyIDBuf)
+	if err != nil {
+		err = errs.ErrEncryptionNewDataKey.Wrap(err).GenWithStack(
+			"fail to generate data key id")
+		return
+	}
+	if n != 8 {
+		err = errs.ErrEncryptionNewDataKey.GenWithStack(
+			"no enough random bytes to generate data key id, bytes %d", n)
+		return
+	}
+	keyID = binary.BigEndian.Uint64(keyIDBuf)
+	keyLength, err := KeyLength(method)
+	if err != nil {
+		return
+	}
+	keyBuf := make([]byte, keyLength)
+	n, err = io.ReadFull(rand.Reader, keyBuf)
+	if err != nil {
+		err = errs.ErrEncryptionNewDataKey.Wrap(err).GenWithStack(
+			"fail to generate data key")
+		return
+	}
+	if n != keyLength {
+		err = errs.ErrEncryptionNewDataKey.GenWithStack(
+			"no enough random bytes to generate data key, bytes %d", n)
+		return
+	}
+	key = &encryptionpb.DataKey{
+		Key:          keyBuf,
+		Method:       method,
+		CreationTime: uint64(time.Now().Unix()),
+		WasExposed:   false,
+	}
+	return
+}
+
+func aesGcmEncryptImpl(
+	key []byte,
+	plaintext []byte,
+	iv IvGcm,
+) (ciphertext []byte, err error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		err = errs.ErrEncryptionGCMEncrypt.Wrap(err).GenWithStack("fail to create aes cipher")
+		return
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		err = errs.ErrEncryptionGCMEncrypt.Wrap(err).GenWithStack("fail to create aes-gcm cipher")
+		return
+	}
+	ciphertext = aesgcm.Seal(nil, iv, plaintext, nil)
+	return
+}
+
+// AesGcmEncrypt encrypt given plaintext with given key using aes256-gcm.
+// The method is used to encrypt data keys.
+func AesGcmEncrypt(
+	key []byte,
+	plaintext []byte,
+) (ciphertext []byte, iv IvGcm, err error) {
+	iv, err = NewIvGcm()
+	if err != nil {
+		return
+	}
+	ciphertext, err = aesGcmEncryptImpl(key, plaintext, iv)
+	return
+}
+
+// AesGcmDecrypt decrypt given ciphertext with given key using aes256-gcm.
+// The method is used to decrypt data keys.
+func AesGcmDecrypt(
+	key []byte,
+	ciphertext []byte,
+	iv IvGcm,
+) (plaintext []byte, err error) {
+	if len(iv) != ivLengthGCM {
+		err = errs.ErrEncryptionGCMDecrypt.GenWithStack("unexpected gcm iv length %d", len(iv))
+		return
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		err = errs.ErrEncryptionGCMDecrypt.Wrap(err).GenWithStack("fail to create aes cipher")
+		return
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		err = errs.ErrEncryptionGCMDecrypt.Wrap(err).GenWithStack("fail to create aes-gcm cipher")
+		return
+	}
+	plaintext, err = aesgcm.Open(nil, iv, ciphertext, nil)
+	if err != nil {
+		err = errs.ErrEncryptionGCMDecrypt.Wrap(err).GenWithStack("authentication fail")
+		return
+	}
+	return
+}

pkg/encryption/crypter_test.go

@@ -0,0 +1,112 @@
+// Copyright 2020 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryption
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "github.com/pingcap/check"
+	"github.com/pingcap/kvproto/pkg/encryptionpb"
+)
+
+func TestCrypter(t *testing.T) {
+	TestingT(t)
+}
+
+type testCrypterSuite struct{}
+
+var _ = Suite(&testCrypterSuite{})
+
+func (s *testCrypterSuite) TestEncryptionMethodSupported(c *C) {
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_PLAINTEXT), Not(IsNil))
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_UNKNOWN), Not(IsNil))
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES128_CTR), IsNil)
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES192_CTR), IsNil)
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES256_CTR), IsNil)
+}
+
+func (s *testCrypterSuite) TestKeyLength(c *C) {
+	_, err := KeyLength(encryptionpb.EncryptionMethod_PLAINTEXT)
+	c.Assert(err, Not(IsNil))
+	_, err = KeyLength(encryptionpb.EncryptionMethod_UNKNOWN)
+	c.Assert(err, Not(IsNil))
+	length, err := KeyLength(encryptionpb.EncryptionMethod_AES128_CTR)
+	c.Assert(err, IsNil)
+	c.Assert(length, Equals, 16)
+	length, err = KeyLength(encryptionpb.EncryptionMethod_AES192_CTR)
+	c.Assert(err, IsNil)
+	c.Assert(length, Equals, 24)
+	length, err = KeyLength(encryptionpb.EncryptionMethod_AES256_CTR)
+	c.Assert(err, IsNil)
+	c.Assert(length, Equals, 32)
+}
+
+func (s *testCrypterSuite) TestNewIv(c *C) {
+	ivCtr, err := NewIvCtr()
+	c.Assert(err, IsNil)
+	c.Assert(len([]byte(ivCtr)), Equals, ivLengthCTR)
+	ivGcm, err := NewIvGcm()
+	c.Assert(err, IsNil)
+	c.Assert(len([]byte(ivGcm)), Equals, ivLengthGCM)
+}
+
+func testNewDataKey(c *C, method encryptionpb.EncryptionMethod) {
+	_, key, err := NewDataKey(method)
+	c.Assert(err, IsNil)
+	length, err := KeyLength(method)
+	c.Assert(err, IsNil)
+	c.Assert(len(key.Key), Equals, length)
+	c.Assert(key.Method, Equals, method)
+	c.Assert(key.WasExposed, IsFalse)
+}
+
+func (s *testCrypterSuite) TestNewDataKey(c *C) {
+	testNewDataKey(c, encryptionpb.EncryptionMethod_AES128_CTR)
+	testNewDataKey(c, encryptionpb.EncryptionMethod_AES192_CTR)
+	testNewDataKey(c, encryptionpb.EncryptionMethod_AES256_CTR)
+}
+
+func (s *testCrypterSuite) TestAesGcmCrypter(c *C) {
+	key, err := hex.DecodeString("ed568fbd8c8018ed2d042a4e5d38d6341486922d401d2022fb81e47c900d3f07")
+	c.Assert(err, IsNil)
+	plaintext, err := hex.DecodeString(
+		"5c873a18af5e7c7c368cb2635e5a15c7f87282085f4b991e84b78c5967e946d4")
+	c.Assert(err, IsNil)
+	// encrypt
+	ivBytes, err := hex.DecodeString("ba432b70336c40c39ba14c1b")
+	c.Assert(err, IsNil)
+	iv := IvGcm(ivBytes)
+	ciphertext, err := aesGcmEncryptImpl(key, plaintext, iv)
+	c.Assert(err, IsNil)
+	c.Assert(len([]byte(iv)), Equals, ivLengthGCM)
+	c.Assert(
+		hex.EncodeToString(ciphertext),
+		Equals,
+		"bbb9b49546350880cf55d4e4eaccc831c506a4aeae7f6cda9c821d4cb8cfc269dcdaecb09592ef25d7a33b40d3f02208",
+	)
+	// decrypt
+	plaintext2, err := AesGcmDecrypt(key, ciphertext, iv)
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(plaintext2, plaintext), IsTrue)
+	// Modify ciphertext to test authentication failure. We modify the beginning of the ciphertext,
+	// which is the real ciphertext part, not the tag.
+	fakeCiphertext := make([]byte, len(ciphertext))
+	copy(fakeCiphertext, ciphertext)
+	// ignore overflow
+	fakeCiphertext[0] = ciphertext[0] + 1
+	_, err = AesGcmDecrypt(key, fakeCiphertext, iv)
+	c.Assert(err, Not(IsNil))
+}

pkg/encryption/key_manager.go

@@ -0,0 +1,25 @@
+// Copyright 2020 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryption
+
+import (
+	"github.com/pingcap/kvproto/pkg/encryptionpb"
+)
+
+// KeyManager maintains the list to encryption keys. It handles encryption key generation and
+// rotation, persisting and loading encryption keys.
+type KeyManager interface {
+	GetCurrentKey() (keyID uint64, key *encryptionpb.DataKey, err error)
+	GetKey(keyID uint64) (key *encryptionpb.DataKey, err error)
+}