Add gRPC over TLS support

Fixes #862
Updates #881

- Make it clear that all three TLS params are required if you supply any in all three modes.
- Update `contour serve` variables to make it clear they're not client certs.
- Remove redundant test now that all three params are required.
- Fix a small linting thing in `cli.go` by adding comments to exported functions.
- Add secure-local Make target
- Add env var support for all the TLS flags.

Signed-off-by: Nick Young <ynick@vmware.com>

.gitignore

@@ -6,3 +6,5 @@ vendor/
 .DS_Store
 go.sum
 localenvoyconfig.yaml
+securelocalenvoyconfig.yaml
+certs/

Makefile

@@ -4,6 +4,8 @@ IMAGE := $(REGISTRY)/$(PROJECT)
 SRCDIRS := ./cmd ./internal ./apis
 PKGS := $(shell GO111MODULE=on go list -mod=readonly ./cmd/... ./internal/...)
 LOCAL_BOOTSTRAP_CONFIG = localenvoyconfig.yaml
+SECURE_LOCAL_BOOTSTRAP_CONFIG = securelocalenvoyconfig.yaml
+
 TAG_LATEST ?= false
 # Used to supply a local Envoy docker container an IP to connect to that is running
 # 'contour serve'. On MacOS this will work, but may not on other OSes. Defining
@@ -45,7 +47,21 @@ ifeq ($(TAG_LATEST), true)
 endif
 
 $(LOCAL_BOOTSTRAP_CONFIG): install
-	contour bootstrap --xds-address $(LOCALIP) --xds-port=8001 $@
+	contour bootstrap --xds-address $(LOCALIP) --xds-port=8001
+
+$(SECURE_LOCAL_BOOTSTRAP_CONFIG): install
+	contour bootstrap --xds-address $(LOCALIP) --xds-port=8001 --envoy-cafile /config/certs/CA.cert --envoy-cert-file /config/certs/client.cert --envoy-key-file /config/certs/client.key $@
+
+secure-local: $(SECURE_LOCAL_BOOTSTRAP_CONFIG)
+	docker run \
+		-it \
+		--mount type=bind,source=$(CURDIR),target=/config \
+		--net bridge \
+		docker.io/envoyproxy/envoy:v1.10.0 \
+		envoy \
+		--config-path /config/$< \
+		--service-node node0 \
+		--service-cluster cluster0
 
 local: $(LOCAL_BOOTSTRAP_CONFIG)
 	docker run \

cmd/contour/cli.go

@@ -15,41 +15,96 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"io/ioutil"
+	"log"
 	"os"
 
-	"github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
 	"github.com/gogo/protobuf/proto"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 )
 
+// Client holds the details for the cli client to connect to.
 type Client struct {
 	ContourAddr string
+	CAFile      string
+	ClientCert  string
+	ClientKey   string
 }
 
 func (c *Client) dial() *grpc.ClientConn {
-	conn, err := grpc.Dial(c.ContourAddr, grpc.WithInsecure())
-	check(err)
+
+	var conn *grpc.ClientConn
+	var err error
+
+	// Check the TLS setup
+	if c.CAFile != "" || c.ClientCert != "" || c.ClientKey != "" {
+		// If one of the three TLS commands is not empty, they all must be not empty
+		if !(c.CAFile != "" && c.ClientCert != "" && c.ClientKey != "") {
+			log.Fatal("You must supply all three TLS parameters - --cafile, --cert-file, --key-file, or none of them.")
+		}
+		// Load the client certificates from disk
+		certificate, err := tls.LoadX509KeyPair(c.ClientCert, c.ClientKey)
+		check(err)
+
+		// Create a certificate pool from the certificate authority
+		certPool := x509.NewCertPool()
+		ca, err := ioutil.ReadFile(c.CAFile)
+		check(err)
+
+		// Append the certificates from the CA
+		if ok := certPool.AppendCertsFromPEM(ca); !ok {
+			// TODO(nyoung) OMG yuck, thanks for this, crypto/tls. Suggestions on alternates welcomed.
+			check(errors.New("failed to append ca certs"))
+		}
+
+		creds := credentials.NewTLS(&tls.Config{
+			// TODO(youngnick): Does this need to be defaulted with a cli flag to
+			// override?
+			// The ServerName here needs to be one of the SANs available in
+			// the serving cert used by contour serve.
+			ServerName:   "contour",
+			Certificates: []tls.Certificate{certificate},
+			RootCAs:      certPool,
+		})
+
+		// Create a connection with the TLS credentials
+		conn, err = grpc.Dial(c.ContourAddr, grpc.WithTransportCredentials(creds))
+		check(err)
+	} else {
+		conn, err = grpc.Dial(c.ContourAddr, grpc.WithInsecure())
+		check(err)
+	}
+
 	return conn
 }
 
+// ClusterStream returns a stream of Clusters using the config in the Client.
 func (c *Client) ClusterStream() v2.ClusterDiscoveryService_StreamClustersClient {
 	stream, err := v2.NewClusterDiscoveryServiceClient(c.dial()).StreamClusters(context.Background())
 	check(err)
 	return stream
 }
 
+// EndpointStream returns a stream of Endpoints using the config in the Client.
 func (c *Client) EndpointStream() v2.ClusterDiscoveryService_StreamClustersClient {
 	stream, err := v2.NewEndpointDiscoveryServiceClient(c.dial()).StreamEndpoints(context.Background())
 	check(err)
 	return stream
 }
 
+// ListenerStream returns a stream of Listeners using the config in the Client.
 func (c *Client) ListenerStream() v2.ClusterDiscoveryService_StreamClustersClient {
 	stream, err := v2.NewListenerDiscoveryServiceClient(c.dial()).StreamListeners(context.Background())
 	check(err)
 	return stream
 }
 
+// RouteStream returns a stream of Routes using the config in the Client.
 func (c *Client) RouteStream() v2.ClusterDiscoveryService_StreamClustersClient {
 	stream, err := v2.NewRouteDiscoveryServiceClient(c.dial()).StreamRoutes(context.Background())
 	check(err)

cmd/contour/contour.go

@@ -14,8 +14,12 @@
 package main
 
 import (
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
@@ -49,19 +53,30 @@ func main() {
 	log := logrus.StandardLogger()
 	app := kingpin.New("contour", "Heptio Contour Kubernetes ingress controller.")
 	var config envoy.BootstrapConfig
+
+	// Set up a zero-valued tls.Config, we'll use this to tell if we need to do
+	// any TLS setup for the 'serve' command.
+	tlsconfig := tls.Config{}
+
 	bootstrap := app.Command("bootstrap", "Generate bootstrap configuration.")
 	path := bootstrap.Arg("path", "Configuration file.").Required().String()
 	bootstrap.Flag("admin-address", "Envoy admin interface address").StringVar(&config.AdminAddress)
 	bootstrap.Flag("admin-port", "Envoy admin interface port").IntVar(&config.AdminPort)
 	bootstrap.Flag("xds-address", "xDS gRPC API address").StringVar(&config.XDSAddress)
 	bootstrap.Flag("xds-port", "xDS gRPC API port").IntVar(&config.XDSGRPCPort)
+	bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load").Envar("ENVOY_CAFILE").StringVar(&config.GrpcCABundle)
+	bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert)
+	bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey)
 
 	// Get the running namespace passed via ENV var from the Kubernetes Downward API
 	config.Namespace = getEnv("CONTOUR_NAMESPACE", "heptio-contour")
 
 	cli := app.Command("cli", "A CLI client for the Heptio Contour Kubernetes ingress controller.")
 	var client Client
 	cli.Flag("contour", "contour host:port.").Default("127.0.0.1:8001").StringVar(&client.ContourAddr)
+	cli.Flag("cafile", "CA bundle file for connecting to a TLS-secured Contour").Envar("CLI_CAFILE").StringVar(&client.CAFile)
+	cli.Flag("cert-file", "Client certificate file for connecting to a TLS-secured Contour").Envar("CLI_CERT_FILE").StringVar(&client.ClientCert)
+	cli.Flag("key-file", "Client key file for connecting to a TLS-secured Contour").Envar("CLI_KEY_FILE").StringVar(&client.ClientKey)
 
 	var resources []string
 	cds := cli.Command("cds", "watch services.")
@@ -82,6 +97,9 @@ func main() {
 	xdsPort := serve.Flag("xds-port", "xDS gRPC API port").Default("8001").Int()
 	statsAddress := serve.Flag("stats-address", "Envoy /stats interface address").Default("0.0.0.0").String()
 	statsPort := serve.Flag("stats-port", "Envoy /stats interface port").Default("8002").Int()
+	caFile := serve.Flag("contour-cafile", "CA bundle file name for serving gRPC with TLS").Envar("CONTOUR_CAFILE").String()
+	contourCert := serve.Flag("contour-cert-file", "Contour certificate file name for serving gRPC over TLS").Envar("CONTOUR_CERT_FILE").String()
+	contourKey := serve.Flag("contour-key-file", "Contour key file name for serving gRPC over TLS").Envar("CONTOUR_KEY_FILE").String()
 
 	ch := contour.CacheHandler{
 		FieldLogger: log.WithField("context", "CacheHandler"),
@@ -158,6 +176,14 @@ func main() {
 		stream := client.RouteStream()
 		watchstream(stream, cache.SecretType, resources)
 	case serve.FullCommand():
+		if *caFile != "" || *contourCert != "" || *contourKey != "" {
+			// If one of the three TLS commands is not empty, they all must be not empty
+			if !(*caFile != "" && *contourCert != "" && *contourKey != "") {
+				log.Fatal("You must supply all three TLS parameters - --contour-cafile, --contour-cert-file, --contour-key-file, or none of them.")
+			}
+			setupTLSConfig(&tlsconfig, *caFile, *contourCert, *contourKey)
+		}
+
 		log.Infof("args: %v", args)
 		var g workgroup.Group
 
@@ -212,9 +238,20 @@ func main() {
 		g.Add(func(stop <-chan struct{}) error {
 			log := log.WithField("context", "grpc")
 			addr := net.JoinHostPort(*xdsAddr, strconv.Itoa(*xdsPort))
-			l, err := net.Listen("tcp", addr)
-			if err != nil {
-				return err
+
+			var l net.Listener
+			var err error
+			if tlsconfig.ClientAuth != tls.NoClientCert {
+				log.Info("Setting up TLS for gRPC")
+				l, err = tls.Listen("tcp", addr, &tlsconfig)
+				if err != nil {
+					return err
+				}
+			} else {
+				l, err = net.Listen("tcp", addr)
+				if err != nil {
+					return err
+				}
 			}
 
 			s := grpc.NewAPI(log, map[string]grpc.Resource{
@@ -296,3 +333,32 @@ func getEnv(key, fallback string) string {
 	}
 	return value
 }
+
+// setupTLSConfig sets up a tls.Config, given cert filenames.
+func setupTLSConfig(config *tls.Config, caFile string, servingCert string, servingKey string) error {
+
+	// First up, load the Contour serving cert and key pair
+
+	cert, err := tls.LoadX509KeyPair(servingCert, servingKey)
+	if err != nil {
+		return err
+	}
+
+	ca, err := ioutil.ReadFile(caFile)
+	if err != nil {
+		return err
+	}
+	certPool := x509.NewCertPool()
+
+	if ok := certPool.AppendCertsFromPEM(ca); !ok {
+		return err
+	}
+
+	config.Certificates = []tls.Certificate{cert}
+	config.ClientAuth = tls.RequireAndVerifyClientCert
+	config.ClientCAs = certPool
+	config.Rand = rand.Reader
+
+	return nil
+
+}

internal/envoy/bootstrap.go

@@ -14,11 +14,13 @@
 package envoy
 
 import (
+	"log"
 	"strconv"
 	"strings"
 	"time"
 
 	api "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	"github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
 	clusterv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2/cluster"
 	"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
@@ -85,9 +87,65 @@ func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap {
 		},
 	}
 
+	if c.GrpcClientCert != "" || c.GrpcClientKey != "" || c.GrpcCABundle != "" {
+		// If one of the two TLS options is not empty, they all must be not empty
+		if !(c.GrpcClientCert != "" && c.GrpcClientKey != "" && c.GrpcCABundle != "") {
+			log.Fatal("You must supply all three TLS parameters - --envoy-cafile, --envoy-cert-file, --envoy-key-file, or none of them.")
+		}
+		b.StaticResources.Clusters[0].TlsContext = grpcUpstreamTLSContext(c.GrpcCABundle, c.GrpcClientCert, c.GrpcClientKey)
+	}
+
 	return b
 }
 
+func grpcUpstreamTLSContext(cafile string, certfile string, keyfile string) *auth.UpstreamTlsContext {
+	if certfile == "" {
+		// Nothig to do
+		return nil
+	}
+
+	if certfile == "" {
+		// Nothing to do
+		return nil
+	}
+
+	if cafile == "" {
+		// You currently must supply a CA file, not just use others.
+		return nil
+	}
+	context := &auth.UpstreamTlsContext{
+		CommonTlsContext: &auth.CommonTlsContext{
+			TlsCertificates: []*auth.TlsCertificate{
+				{
+					CertificateChain: &core.DataSource{
+						Specifier: &core.DataSource_Filename{
+							Filename: certfile,
+						},
+					},
+					PrivateKey: &core.DataSource{
+						Specifier: &core.DataSource_Filename{
+							Filename: keyfile,
+						},
+					},
+				},
+			},
+			ValidationContextType: &auth.CommonTlsContext_ValidationContext{
+				ValidationContext: &auth.CertificateValidationContext{
+					TrustedCa: &core.DataSource{
+						Specifier: &core.DataSource_Filename{
+							Filename: cafile,
+						},
+					},
+					// TODO(youngnick): Does there need to be a flag wired down to here?
+					VerifySubjectAltName: []string{"contour"},
+				},
+			},
+		},
+	}
+
+	return context
+}
+
 func stringOrDefault(s, def string) string {
 	if s == "" {
 		return def
@@ -126,4 +184,15 @@ type BootstrapConfig struct {
 
 	// Namespace is the namespace where Contour is running
 	Namespace string
+
+	//GrpcCABundle is the filename that contains a CA certificate chain that can
+	//verify the client cert.
+	GrpcCABundle string
+
+	// GrpcClientCert is the filename that contains a client certificate. May contain a full bundle if you
+	// don't want to pass a CA Bundle.
+	GrpcClientCert string
+
+	// GrpcClientKey is the filename that contains a client key for secure gRPC with TLS.
+	GrpcClientKey string
 }