security: Encrypt region boundary keys, Part 1 - helper functions

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b
 	github.com/pingcap/log v0.0.0-20200511115504-543df19646ad
 	github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/common v0.4.1
 	github.com/sasha-s/go-deadlock v0.2.0
@@ -54,3 +55,5 @@ require (
 )
 
 replace go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.5
+
+replace github.com/pingcap/kvproto => github.com/yiwu-arbug/kvproto v0.0.0-20200909020201-e9b87ecd4a2d

go.sum

@@ -126,7 +126,6 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/goccy/go-graphviz v0.0.5 h1:qcjgvNiYbLyfLAq9LvyYBJ7sNMbQh9w4FoAzBDrYhYw=
 github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk=
-github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -141,7 +140,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -296,10 +294,6 @@ github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de h1:mW8hC2yXTpflf
 github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ=
 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+v8Jlc98uMBvKIzr1a+UhnLyVYn8Q5Q=
 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
-github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
-github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b h1:WFBUlCITLWmU7h2d1NS29s2xvbiF/+o37OcW92iIRac=
-github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
@@ -412,6 +406,8 @@ github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37w
 github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yiwu-arbug/kvproto v0.0.0-20200909020201-e9b87ecd4a2d h1:05aox/RarfRL4W4b8WXdp6caWc5KRwA/3PXACDCnYdU=
+github.com/yiwu-arbug/kvproto v0.0.0-20200909020201-e9b87ecd4a2d/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ=
 github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -572,12 +568,10 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo=
 google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=

pkg/encryption/crypter.go

@@ -0,0 +1,193 @@
+// Copyright 2020 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryption
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/binary"
+	"io"
+	"time"
+
+	"github.com/pingcap/kvproto/pkg/encryptionpb"
+	"github.com/pkg/errors"
+)
+
+const (
+	ivLengthCTR = 16
+	ivLengthGCM = 12
+)
+
+// CheckEncryptionMethodSupported check whether the encryption method is currently supported.
+// This is to handle future extension to encryption methods on kvproto side.
+func CheckEncryptionMethodSupported(method encryptionpb.EncryptionMethod) error {
+	switch method {
+	case encryptionpb.EncryptionMethod_AES128_CTR:
+		return nil
+	case encryptionpb.EncryptionMethod_AES192_CTR:
+		return nil
+	case encryptionpb.EncryptionMethod_AES256_CTR:
+		return nil
+	default:
+		name, ok := encryptionpb.EncryptionMethod_name[int32(method)]
+		if ok {
+			return errors.Errorf("invalid encryption method %s", name)
+		}
+		return errors.Errorf("invalid encryption method %d", int32(method))
+	}
+}
+
+// KeyLength return the encryption key lenght for supported encryption methods.
+func KeyLength(method encryptionpb.EncryptionMethod) int {
+	switch method {
+	case encryptionpb.EncryptionMethod_AES128_CTR:
+		return 16
+	case encryptionpb.EncryptionMethod_AES192_CTR:
+		return 24
+	case encryptionpb.EncryptionMethod_AES256_CTR:
+		return 32
+	default:
+		panic("unsupported encryption method")
+	}
+}
+
+// IvCtr represent IV bytes for CTR mode.
+type IvCtr []byte
+
+// IvGcm represent IV bytes for GCM mode.
+type IvGcm []byte
+
+func newIV(ivLength int) ([]byte, error) {
+	iv := make([]byte, ivLength)
+	n, err := io.ReadFull(rand.Reader, iv)
+	if err != nil {
+		return nil, errors.Wrap(err, "fail to generate iv")
+	}
+	if n != ivLength {
+		return nil, errors.New("no enough random bytes to generate iv")
+	}
+	return iv, nil
+}
+
+// NewIvCtr randomly generate an IV for CTR mode.
+func NewIvCtr() (IvCtr, error) {
+	return newIV(ivLengthCTR)
+}
+
+// NewIvCtr randomly generate an IV for GCM mode.
+func NewIvGcm() (IvGcm, error) {
+	return newIV(ivLengthGCM)
+}
+
+// NewDataKey randomly generate a new data key.
+func NewDataKey(
+	method encryptionpb.EncryptionMethod,
+) (keyId uint64, key *encryptionpb.DataKey, err error) {
+	err = CheckEncryptionMethodSupported(method)
+	if err != nil {
+		return
+	}
+	keyIdBuf := make([]byte, 8)
+	n, err := io.ReadFull(rand.Reader, keyIdBuf)
+	if err != nil {
+		err = errors.Wrap(err, "fail to generate data key id")
+		return
+	}
+	if n != 8 {
+		err = errors.New("no enough random bytes to generate data key id")
+		return
+	}
+	keyId = binary.BigEndian.Uint64(keyIdBuf)
+	keyLength := KeyLength(method)
+	keyBuf := make([]byte, keyLength)
+	n, err = io.ReadFull(rand.Reader, keyBuf)
+	if err != nil {
+		err = errors.Wrap(err, "fail to generate data key")
+		return
+	}
+	if n != keyLength {
+		err = errors.New("no enough random bytes to generate data key")
+		return
+	}
+	key = &encryptionpb.DataKey{
+		Key:          keyBuf,
+		Method:       method,
+		CreationTime: uint64(time.Now().Unix()),
+		WasExposed:   false,
+	}
+	return
+}
+
+func aesGcmEncryptImpl(
+	key []byte,
+	plaintext []byte,
+	iv IvGcm,
+) (ciphertext []byte, err error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		err = errors.Wrap(err, "fail to create aes cipher")
+		return
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		err = errors.Wrap(err, "fail to create aes-gcm cipher")
+		return
+	}
+	ciphertext = aesgcm.Seal(nil, iv, plaintext, nil)
+	return
+}
+
+// AesGcmEncrypt encrypt given plaintext with given key using aes256-gcm.
+// The method is used to encrypt data keys.
+func AesGcmEncrypt(
+	key []byte,
+	plaintext []byte,
+) (ciphertext []byte, iv IvGcm, err error) {
+	iv, err = NewIvGcm()
+	if err != nil {
+		return
+	}
+	ciphertext, err = aesGcmEncryptImpl(key, plaintext, iv)
+	return
+}
+
+// AesGcmEncrypt encrypt given plaintext with given key using aes256-gcm.
+// The method is used to decrypt data keys.
+func AesGcmDecrypt(
+	key []byte,
+	ciphertext []byte,
+	iv IvGcm,
+) (plaintext []byte, err error) {
+	if len(iv) != ivLengthGCM {
+		err = errors.Errorf("unexpected gcm iv length %d", len(iv))
+		return
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		err = errors.Wrap(err, "fail to create aes cipher")
+		return
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		err = errors.Wrap(err, "fail to create aes-gcm cipher")
+		return
+	}
+	plaintext, err = aesgcm.Open(nil, iv, ciphertext, nil)
+	if err != nil {
+		err = errors.Wrap(err, "authentication fail")
+		return
+	}
+	return
+}

pkg/encryption/crypter_test.go

@@ -0,0 +1,100 @@
+// Copyright 2020 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryption
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "github.com/pingcap/check"
+	"github.com/pingcap/kvproto/pkg/encryptionpb"
+)
+
+func TestCrypter(t *testing.T) {
+	TestingT(t)
+}
+
+type testCrypterSuite struct{}
+
+var _ = Suite(&testCrypterSuite{})
+
+func (s *testCrypterSuite) TestEncryptionMethodSupported(c *C) {
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_PLAINTEXT), Not(IsNil))
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_UNKNOWN), Not(IsNil))
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES128_CTR), IsNil)
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES192_CTR), IsNil)
+	c.Assert(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES256_CTR), IsNil)
+}
+
+func (s *testCrypterSuite) TestKeyLength(c *C) {
+	c.Assert(KeyLength(encryptionpb.EncryptionMethod_AES128_CTR), Equals, 16)
+	c.Assert(KeyLength(encryptionpb.EncryptionMethod_AES192_CTR), Equals, 24)
+	c.Assert(KeyLength(encryptionpb.EncryptionMethod_AES256_CTR), Equals, 32)
+}
+
+func (s *testCrypterSuite) TestNewIv(c *C) {
+	ivCtr, err := NewIvCtr()
+	c.Assert(err, IsNil)
+	c.Assert(len([]byte(ivCtr)), Equals, ivLengthCTR)
+	ivGcm, err := NewIvGcm()
+	c.Assert(err, IsNil)
+	c.Assert(len([]byte(ivGcm)), Equals, ivLengthGCM)
+}
+
+func testNewDataKey(c *C, method encryptionpb.EncryptionMethod) {
+	_, key, err := NewDataKey(method)
+	c.Assert(err, IsNil)
+	c.Assert(len(key.Key), Equals, KeyLength(method))
+	c.Assert(key.Method, Equals, method)
+	c.Assert(key.WasExposed, IsFalse)
+}
+
+func (s *testCrypterSuite) TestNewDataKey(c *C) {
+	testNewDataKey(c, encryptionpb.EncryptionMethod_AES128_CTR)
+	testNewDataKey(c, encryptionpb.EncryptionMethod_AES192_CTR)
+	testNewDataKey(c, encryptionpb.EncryptionMethod_AES256_CTR)
+}
+
+func (s *testCrypterSuite) TestAesGcmCrypter(c *C) {
+	key, err := hex.DecodeString("ed568fbd8c8018ed2d042a4e5d38d6341486922d401d2022fb81e47c900d3f07")
+	c.Assert(err, IsNil)
+	plaintext, err := hex.DecodeString(
+		"5c873a18af5e7c7c368cb2635e5a15c7f87282085f4b991e84b78c5967e946d4")
+	c.Assert(err, IsNil)
+	// encrypt
+	ivBytes, err := hex.DecodeString("ba432b70336c40c39ba14c1b")
+	c.Assert(err, IsNil)
+	iv := IvGcm(ivBytes)
+	ciphertext, err := aesGcmEncryptImpl(key, plaintext, iv)
+	c.Assert(err, IsNil)
+	c.Assert(len([]byte(iv)), Equals, ivLengthGCM)
+	c.Assert(
+		hex.EncodeToString(ciphertext),
+		Equals,
+		"bbb9b49546350880cf55d4e4eaccc831c506a4aeae7f6cda9c821d4cb8cfc269dcdaecb09592ef25d7a33b40d3f02208",
+	)
+	// decrypt
+	plaintext2, err := AesGcmDecrypt(key, ciphertext, iv)
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(plaintext2, plaintext), IsTrue)
+	// Modify ciphertext to test authentication failure. We modify the beginning of the ciphertext,
+	// which is the real ciphertext part, not the tag.
+	fake_ciphertext := make([]byte, len(ciphertext))
+	copy(fake_ciphertext, ciphertext)
+	// ignore overflow
+	fake_ciphertext[0] = ciphertext[0] + 1
+	_, err = AesGcmDecrypt(key, fake_ciphertext, iv)
+	c.Assert(err, Not(IsNil))
+}

pkg/encryption/key_manager.go

@@ -0,0 +1,23 @@
+// Copyright 2020 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryption
+
+import (
+	"github.com/pingcap/kvproto/pkg/encryptionpb"
+)
+
+type KeyManager interface {
+	GetCurrentKey() (keyId uint64, key *encryptionpb.DataKey, err error)
+	GetKey(keyId uint64) (key *encryptionpb.DataKey, err error)
+}