Skip to content

Commit

Permalink
Add option for TLS enabled etcd server connections. (#5496)
Browse files Browse the repository at this point in the history 
* Added functionality for etcd tls support, and included tests.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Almost got tests working for tls etcd

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Changed to pass by value per review suggestion.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Removed extra safety checks per review suggestion.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Changed error return per review suggestion.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Update comments per review suggestion.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Moved common test logic into main tlstest file for access by other tests.  Updated server tests to use helper, and verified we are now working with https

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Update go/vt/topo/etcd2topo/server.go

Co-Authored-By: Anthony Yeh <enisoc@planetscale.com>
Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Updated etcd2 flag names to etcd per convention as pointed out by reviewer.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Bring todo inside func so it doesn't get added into auto-generated docs

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Removed client.close() from defer as this should get called automatically by server object when server.close() is called

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Fixed up defers per review suggestions.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Switched over to using server public api instead of pulling client out of it in tests.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Made sure that we retry until we establish an initial client, and close that client at the end of the function that establishes an etcd server for testing.

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Adding logging to try to figure out why tests are failing

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Setup stderr and stdout for debugging in cicd

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Updating helper function to use DNS compatible common name

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Corrected log for accuracy

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Corrected fatals message formatting

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Adding dns compliant server common name

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Changing these back so we break with logging in CICD. :-(

Signed-off-by: Peter Farr <Peter@PrismaPhonic.com>

* Pin unit test job to a specific version of etcd.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
  • Loading branch information
@PrismaPhonic @enisoc
PrismaPhonic authored and enisoc committed Dec 13, 2019
1 parent d8c6d2a commit e140145
Show file tree
Hide file tree
Showing 6 changed files with 245 additions and 81 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/unit.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@ jobs:
- name: Get dependencies
run: |
sudo apt-get update
sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget ant openjdk-8-jdk
sudo apt-get install -y mysql-server mysql-client make unzip g++ curl git wget ant openjdk-8-jdk
sudo service mysql stop
sudo service etcd stop
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
mkdir -p dist bin
curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
go mod download
- name: Run make tools
Expand Down
3 changes: 3 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (
github.com/golang/mock v1.3.1
github.com/golang/protobuf v1.3.2
github.com/golang/snappy v0.0.0-20170215233205-553a64147049
github.com/google/btree v1.0.0 // indirect
github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect
github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
Expand All @@ -49,6 +50,8 @@ require (
github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984
github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02
github.com/opentracing/opentracing-go v1.1.0
Expand Down
48 changes: 48 additions & 0 deletions go/vt/tlstest/tlstest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,51 @@ func CreateSignedCert(root, parent, serial, name, commonName string) {
"-extfile", config,
"-out", cert)
}

type ClientServerKeyPairs struct {
ServerCert string
ServerKey string
ServerCA string
ServerName string
ClientCert string
ClientKey string
ClientCA string
}

var serialCounter = 0

func CreateClientServerCertPairs(root string) ClientServerKeyPairs {
// Create the certs and configs.
CreateCA(root)

serverSerial := fmt.Sprintf("%03d", serialCounter*2+1)
clientSerial := fmt.Sprintf("%03d", serialCounter*2+2)

serialCounter = serialCounter + 1

serverName := fmt.Sprintf("server-%s", serverSerial)
serverCACommonName := fmt.Sprintf("Server %s CA", serverSerial)
serverCertName := fmt.Sprintf("server-instance-%s", serverSerial)
serverCertCommonName := fmt.Sprintf("server%s.example.com", serverSerial)

clientName := fmt.Sprintf("clients-%s", serverSerial)
clientCACommonName := fmt.Sprintf("Clients %s CA", serverSerial)
clientCertName := fmt.Sprintf("client-instance-%s", serverSerial)
clientCertCommonName := fmt.Sprintf("Client Instance %s", serverSerial)

CreateSignedCert(root, CA, serverSerial, serverName, serverCACommonName)
CreateSignedCert(root, serverName, serverSerial, serverCertName, serverCertCommonName)

CreateSignedCert(root, CA, clientSerial, clientName, clientCACommonName)
CreateSignedCert(root, clientName, serverSerial, clientCertName, clientCertCommonName)

return ClientServerKeyPairs{
ServerCert: path.Join(root, fmt.Sprintf("%s-cert.pem", serverCertName)),
ServerKey: path.Join(root, fmt.Sprintf("%s-key.pem", serverCertName)),
ServerCA: path.Join(root, fmt.Sprintf("%s-cert.pem", serverName)),
ClientCert: path.Join(root, fmt.Sprintf("%s-cert.pem", clientCertName)),
ClientKey: path.Join(root, fmt.Sprintf("%s-key.pem", clientCertName)),
ClientCA: path.Join(root, fmt.Sprintf("%s-cert.pem", clientName)),
ServerName: serverCertCommonName,
}
}
99 changes: 24 additions & 75 deletions go/vt/tlstest/tlstest_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"io/ioutil"
"net"
"os"
"path"
"strings"
"sync"
"testing"
Expand All @@ -47,20 +46,20 @@ func TestClientServer(t *testing.T) {
}
defer os.RemoveAll(root)

clientServerKeyPairs := createClientServerCertPairs(root)
clientServerKeyPairs := CreateClientServerCertPairs(root)

serverConfig, err := vttls.ServerConfig(
clientServerKeyPairs.serverCert,
clientServerKeyPairs.serverKey,
clientServerKeyPairs.clientCA)
clientServerKeyPairs.ServerCert,
clientServerKeyPairs.ServerKey,
clientServerKeyPairs.ClientCA)
if err != nil {
t.Fatalf("TLSServerConfig failed: %v", err)
}
clientConfig, err := vttls.ClientConfig(
clientServerKeyPairs.clientCert,
clientServerKeyPairs.clientKey,
clientServerKeyPairs.serverCA,
clientServerKeyPairs.serverName)
clientServerKeyPairs.ClientCert,
clientServerKeyPairs.ClientKey,
clientServerKeyPairs.ServerCA,
clientServerKeyPairs.ServerName)
if err != nil {
t.Fatalf("TLSClientConfig failed: %v", err)
}
Expand Down Expand Up @@ -117,10 +116,10 @@ func TestClientServer(t *testing.T) {
//

badClientConfig, err := vttls.ClientConfig(
clientServerKeyPairs.serverCert,
clientServerKeyPairs.serverKey,
clientServerKeyPairs.serverCA,
clientServerKeyPairs.serverName)
clientServerKeyPairs.ServerCert,
clientServerKeyPairs.ServerKey,
clientServerKeyPairs.ServerCA,
clientServerKeyPairs.ServerName)
if err != nil {
t.Fatalf("TLSClientConfig failed: %v", err)
}
Expand Down Expand Up @@ -165,69 +164,19 @@ func TestClientServer(t *testing.T) {
}
}

var serialCounter = 0

type clientServerKeyPairs struct {
serverCert string
serverKey string
serverCA string
serverName string
clientCert string
clientKey string
clientCA string
}

func createClientServerCertPairs(root string) clientServerKeyPairs {

// Create the certs and configs.
CreateCA(root)

serverSerial := fmt.Sprintf("%03d", serialCounter*2+1)
clientSerial := fmt.Sprintf("%03d", serialCounter*2+2)

serialCounter = serialCounter + 1

serverName := fmt.Sprintf("server-%s", serverSerial)
serverCACommonName := fmt.Sprintf("Server %s CA", serverSerial)
serverCertName := fmt.Sprintf("server-instance-%s", serverSerial)
serverCertCommonName := fmt.Sprintf("server%s.example.com", serverSerial)

clientName := fmt.Sprintf("clients-%s", serverSerial)
clientCACommonName := fmt.Sprintf("Clients %s CA", serverSerial)
clientCertName := fmt.Sprintf("client-instance-%s", serverSerial)
clientCertCommonName := fmt.Sprintf("Client Instance %s", serverSerial)

CreateSignedCert(root, CA, serverSerial, serverName, serverCACommonName)
CreateSignedCert(root, serverName, serverSerial, serverCertName, serverCertCommonName)

CreateSignedCert(root, CA, clientSerial, clientName, clientCACommonName)
CreateSignedCert(root, clientName, serverSerial, clientCertName, clientCertCommonName)

return clientServerKeyPairs{
serverCert: path.Join(root, fmt.Sprintf("%s-cert.pem", serverCertName)),
serverKey: path.Join(root, fmt.Sprintf("%s-key.pem", serverCertName)),
serverCA: path.Join(root, fmt.Sprintf("%s-cert.pem", serverName)),
clientCert: path.Join(root, fmt.Sprintf("%s-cert.pem", clientCertName)),
clientKey: path.Join(root, fmt.Sprintf("%s-key.pem", clientCertName)),
clientCA: path.Join(root, fmt.Sprintf("%s-cert.pem", clientName)),
serverName: serverCertCommonName,
}

}

func getServerConfig(keypairs clientServerKeyPairs) (*tls.Config, error) {
func getServerConfig(keypairs ClientServerKeyPairs) (*tls.Config, error) {
return vttls.ServerConfig(
keypairs.clientCert,
keypairs.clientKey,
keypairs.serverCA)
keypairs.ClientCert,
keypairs.ClientKey,
keypairs.ServerCA)
}

func getClientConfig(keypairs clientServerKeyPairs) (*tls.Config, error) {
func getClientConfig(keypairs ClientServerKeyPairs) (*tls.Config, error) {
return vttls.ClientConfig(
keypairs.clientCert,
keypairs.clientKey,
keypairs.serverCA,
keypairs.serverName)
keypairs.ClientCert,
keypairs.ClientKey,
keypairs.ServerCA,
keypairs.ServerName)
}

func TestServerTLSConfigCaching(t *testing.T) {
Expand All @@ -242,7 +191,7 @@ func TestClientTLSConfigCaching(t *testing.T) {
})
}

func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(clientServerKeyPairs) (*tls.Config, error), getCertPool func(tlsConfig *tls.Config) *x509.CertPool) {
func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(ClientServerKeyPairs) (*tls.Config, error), getCertPool func(tlsConfig *tls.Config) *x509.CertPool) {
// Our test root.
root, err := ioutil.TempDir("", rootPrefix)
if err != nil {
Expand All @@ -252,8 +201,8 @@ func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(c

const configsToGenerate = 1

firstClientServerKeyPairs := createClientServerCertPairs(root)
secondClientServerKeyPairs := createClientServerCertPairs(root)
firstClientServerKeyPairs := CreateClientServerCertPairs(root)
secondClientServerKeyPairs := CreateClientServerCertPairs(root)

firstExpectedConfig, _ := generateConfig(firstClientServerKeyPairs)
secondExpectedConfig, _ := generateConfig(secondClientServerKeyPairs)
Expand Down
42 changes: 38 additions & 4 deletions go/vt/topo/etcd2topo/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,21 @@ We follow these conventions within this package:
package etcd2topo

import (
"flag"
"strings"
"time"

"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/pkg/transport"
"vitess.io/vitess/go/vt/topo"
)

var (
clientCertPath = flag.String("topo_etcd_tls_cert", "", "path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS")
clientKeyPath = flag.String("topo_etcd_tls_key", "", "path to the client key to use to connect to the etcd topo server, enables TLS")
serverCaPath = flag.String("topo_etcd_tls_ca", "", "path to the ca to use to validate the server cert when connecting to the etcd topo server")
)

// Factory is the consul topo.Factory implementation.
type Factory struct{}

Expand Down Expand Up @@ -71,12 +79,31 @@ func (s *Server) Close() {
s.cli = nil
}

// NewServer returns a new etcdtopo.Server.
func NewServer(serverAddr, root string) (*Server, error) {
cli, err := clientv3.New(clientv3.Config{
func NewServerWithOpts(serverAddr, root, certPath, keyPath, caPath string) (*Server, error) {
// TODO: Rename this to NewServer and change NewServer to a name that signifies it uses the process-wide TLS settings.

config := clientv3.Config{
Endpoints: strings.Split(serverAddr, ","),
DialTimeout: 5 * time.Second,
})
}

// If TLS is enabled, attach TLS config info.
if certPath != "" && keyPath != "" {
// Safe now to build up TLS info.
tlsInfo := transport.TLSInfo{
CertFile: certPath,
KeyFile: keyPath,
TrustedCAFile: caPath,
}

tlsConfig, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
}
config.TLS = tlsConfig
}

cli, err := clientv3.New(config)
if err != nil {
return nil, err
}
Expand All @@ -87,6 +114,13 @@ func NewServer(serverAddr, root string) (*Server, error) {
}, nil
}

// NewServer returns a new etcdtopo.Server.
func NewServer(serverAddr, root string) (*Server, error) {
// TODO: Rename this to a name to signifies this function uses the process-wide TLS settings.

return NewServerWithOpts(serverAddr, root, *clientCertPath, *clientKeyPath, *serverCaPath)
}

func init() {
topo.RegisterFactory("etcd2", Factory{})
}
Loading

0 comments on commit e140145

Please sign in to comment.