client: support pass cert bytes in SecurityOption (#4203)

* client: support pass cert bytes in SecurityOption

Signed-off-by: ehco1996 <zh19960202@gmail.com>

* add more test

Signed-off-by: ehco1996 <zh19960202@gmail.com>

* fix lint

Signed-off-by: ehco1996 <zh19960202@gmail.com>

* address comment

Signed-off-by: ehco1996 <zh19960202@gmail.com>

* address comment

Signed-off-by: ehco1996 <zh19960202@gmail.com>

Co-authored-by: Ti Chi Robot <ti-community-prow-bot@tidb.io>

client/base_client.go

@@ -66,6 +66,10 @@ type SecurityOption struct {
 	CAPath   string
 	CertPath string
 	KeyPath  string
+
+	SSLCABytes   []byte
+	SSLCertBytes []byte
+	SSLKEYBytes  []byte
 }
 
 // newBaseClient returns a new baseClient.
@@ -385,6 +389,10 @@ func (c *baseClient) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error)
 		CAPath:   c.security.CAPath,
 		CertPath: c.security.CertPath,
 		KeyPath:  c.security.KeyPath,
+
+		SSLCABytes:   c.security.SSLCABytes,
+		SSLCertBytes: c.security.SSLCertBytes,
+		SSLKEYBytes:  c.security.SSLKEYBytes,
 	}.ToTLSConfig()
 	if err != nil {
 		return nil, err

errors.toml

@@ -126,6 +126,16 @@ error = '''
 wrong range keys
 '''
 
+["PD:crypto:ErrCryptoAppendCertsFromPEM"]
+error = '''
+cert pool append certs error
+'''
+
+["PD:crypto:ErrCryptoX509KeyPair"]
+error = '''
+x509 keypair error
+'''
+
 ["PD:dashboard:ErrDashboardStart"]
 error = '''
 start dashboard failed

pkg/errs/errno.go

@@ -307,3 +307,9 @@ var (
 	ErrEncryptionSaveDataKeys       = errors.Normalize("failed to save data keys", errors.RFCCodeText("PD:encryption:ErrEncryptionSaveDataKeys"))
 	ErrEncryptionKMS                = errors.Normalize("KMS error", errors.RFCCodeText("PD:ErrEncryptionKMS"))
 )
+
+// crypto
+var (
+	ErrCryptoX509KeyPair        = errors.Normalize("x509 keypair error", errors.RFCCodeText("PD:crypto:ErrCryptoX509KeyPair"))
+	ErrCryptoAppendCertsFromPEM = errors.Normalize("cert pool append certs error", errors.RFCCodeText("PD:crypto:ErrCryptoAppendCertsFromPEM"))
+)

pkg/grpcutil/grpcutil.go

@@ -17,6 +17,7 @@ package grpcutil
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"net/url"
 
 	"github.com/pingcap/log"
@@ -40,10 +41,33 @@ type TLSConfig struct {
 	KeyPath string `toml:"key-path" json:"key-path"`
 	// CertAllowedCN is a CN which must be provided by a client
 	CertAllowedCN []string `toml:"cert-allowed-cn" json:"cert-allowed-cn"`
+
+	SSLCABytes   []byte
+	SSLCertBytes []byte
+	SSLKEYBytes  []byte
 }
 
 // ToTLSConfig generates tls config.
 func (s TLSConfig) ToTLSConfig() (*tls.Config, error) {
+	if len(s.SSLCABytes) != 0 || len(s.SSLCertBytes) != 0 || len(s.SSLKEYBytes) != 0 {
+		cert, err := tls.X509KeyPair(s.SSLCertBytes, s.SSLKEYBytes)
+		if err != nil {
+			return nil, errs.ErrCryptoX509KeyPair.GenWithStackByCause()
+		}
+		certificates := []tls.Certificate{cert}
+		// Create a certificate pool from CA
+		certPool := x509.NewCertPool()
+		// Append the certificates from the CA
+		if !certPool.AppendCertsFromPEM(s.SSLCABytes) {
+			return nil, errs.ErrCryptoAppendCertsFromPEM.GenWithStackByCause()
+		}
+		return &tls.Config{
+			Certificates: certificates,
+			RootCAs:      certPool,
+			NextProtos:   []string{"h2", "http/1.1"}, // specify `h2` to let Go use HTTP/2.
+		}, nil
+	}
+
 	if len(s.CertPath) == 0 && len(s.KeyPath) == 0 {
 		return nil, nil
 	}

pkg/grpcutil/grpcutil_test.go

@@ -0,0 +1,59 @@
+package grpcutil
+
+import (
+	"os"
+	"testing"
+
+	. "github.com/pingcap/check"
+	"github.com/pingcap/errors"
+	"github.com/tikv/pd/pkg/errs"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+var _ = Suite(&gRPCUtilSuite{})
+
+type gRPCUtilSuite struct{}
+
+func loadTLSContent(c *C, caPath, certPath, keyPath string) (caData, certData, keyData []byte) {
+	var err error
+	caData, err = os.ReadFile(caPath)
+	c.Assert(err, IsNil)
+	certData, err = os.ReadFile(certPath)
+	c.Assert(err, IsNil)
+	keyData, err = os.ReadFile(keyPath)
+	c.Assert(err, IsNil)
+	return
+}
+
+func (s *gRPCUtilSuite) TestToTLSConfig(c *C) {
+	tlsConfig := TLSConfig{
+		KeyPath:  "../../tests/client/cert/pd-server-key.pem",
+		CertPath: "../../tests/client/cert/pd-server.pem",
+		CAPath:   "../../tests/client/cert/ca.pem",
+	}
+	// test without bytes
+	_, err := tlsConfig.ToTLSConfig()
+	c.Assert(err, IsNil)
+
+	// test with bytes
+	caData, certData, keyData := loadTLSContent(c, tlsConfig.CAPath, tlsConfig.CertPath, tlsConfig.KeyPath)
+	tlsConfig.SSLCABytes = caData
+	tlsConfig.SSLCertBytes = certData
+	tlsConfig.SSLKEYBytes = keyData
+	_, err = tlsConfig.ToTLSConfig()
+	c.Assert(err, IsNil)
+
+	// test wrong cert bytes
+	tlsConfig.SSLCertBytes = []byte("invalid cert")
+	_, err = tlsConfig.ToTLSConfig()
+	c.Assert(errors.ErrorEqual(err, errs.ErrCryptoX509KeyPair), IsTrue)
+
+	//test wrong ca bytes
+	tlsConfig.SSLCertBytes = certData
+	tlsConfig.SSLCABytes = []byte("invalid ca")
+	_, err = tlsConfig.ToTLSConfig()
+	c.Assert(errors.ErrorEqual(err, errs.ErrCryptoAppendCertsFromPEM), IsTrue)
+}

tests/client/client_tls_test.go

@@ -189,6 +189,29 @@ func (s *clientTLSTestSuite) testTLSReload(
 	c.Assert(err, IsNil)
 	dcancel()
 	cli.Close()
+
+	// 7. test use raw bytes to init tls config
+	caData, certData, keyData := loadTLSContent(c,
+		testClientTLSInfo.TrustedCAFile, testClientTLSInfo.CertFile, testClientTLSInfo.KeyFile)
+	ctx1, cancel1 := context.WithTimeout(s.ctx, 2*time.Second)
+	_, err = pd.NewClientWithContext(ctx1, endpoints, pd.SecurityOption{
+		SSLCABytes:   caData,
+		SSLCertBytes: certData,
+		SSLKEYBytes:  keyData,
+	}, pd.WithGRPCDialOptions(grpc.WithBlock()))
+	c.Assert(err, IsNil)
+	cancel1()
+}
+
+func loadTLSContent(c *C, caPath, certPath, keyPath string) (caData, certData, keyData []byte) {
+	var err error
+	caData, err = os.ReadFile(caPath)
+	c.Assert(err, IsNil)
+	certData, err = os.ReadFile(certPath)
+	c.Assert(err, IsNil)
+	keyData, err = os.ReadFile(keyPath)
+	c.Assert(err, IsNil)
+	return
 }
 
 // copyTLSFiles clones certs files to dst directory.