Add gRPC over TLS support

Fixes projectcontour#862
Updates projectcontour#881

Signed-off-by: Nick Young <ynick@vmware.com>

cmd/contour/cli.go

@@ -15,20 +15,72 @@ package main
 
 import (
  "context"
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "io/ioutil"
+ "log"
  "os"
 
- "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+ v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
  "github.com/gogo/protobuf/proto"
  "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
 )
 
+// Client holds the details for the cli client to connect to.
 type Client struct {
  ContourAddr string
+ CAFile string
+ ClientCert string
+ ClientKey string
 }
 
 func (c *Client) dial() *grpc.ClientConn {
- conn, err := grpc.Dial(c.ContourAddr, grpc.WithInsecure())
- check(err)
+
+ var conn *grpc.ClientConn
+ var err error
+
+ // Check the TLS setup
+ if c.CAFile != "" || c.ClientCert != "" || c.ClientKey != "" {
+ // If one of the three TLS commands is not empty, they all must be not empty
+ if !(c.CAFile != "" && c.ClientCert != "" && c.ClientKey != "") {
+ log.Fatal("Can't have some TLS config, it's all or none")
+ }
+ // Load the client certificates from disk
+ certificate, err := tls.LoadX509KeyPair(c.ClientCert, c.ClientKey)
+ check(err)
+
+ // Create a certificate pool from the certificate authority
+ certPool := x509.NewCertPool()
+ ca, err := ioutil.ReadFile(c.CAFile)
+ check(err)
+
+ // Append the certificates from the CA
+ if ok := certPool.AppendCertsFromPEM(ca); !ok {
+ // TODO(nyoung) OMG yuck.
+ check(errors.New("failed to append ca certs"))
+ }
+
+ creds := credentials.NewTLS(&tls.Config{
+ // ServerName: c.ContourAddr, // NOTE: this is required!
+ // TODO(youngnick): this needs to be defaulted with a cli flag to
+ // override
+ // The ServerName here needs to be one of the SANs available in
+ // the serving cert used by contour serve.
+ ServerName: "contour", // NOTE: this is required!
+ Certificates: []tls.Certificate{certificate},
+ RootCAs: certPool,
+ })
+
+ // Create a connection with the TLS credentials
+ conn, err = grpc.Dial(c.ContourAddr, grpc.WithTransportCredentials(creds))
+ check(err)
+ } else {
+ conn, err = grpc.Dial(c.ContourAddr, grpc.WithInsecure())
+ check(err)
+ }
+
  return conn
 }
 

cmd/contour/contour.go

@@ -14,8 +14,12 @@
 package main
 
 import (
+ "crypto/rand"
+ "crypto/tls"
+ "crypto/x509"
  "flag"
  "fmt"
+ "io/ioutil"
  "net"
  "os"
  "path/filepath"
@@ -49,19 +53,30 @@ func main() {
  log := logrus.StandardLogger()
  app := kingpin.New("contour", "Heptio Contour Kubernetes ingress controller.")
  var config envoy.BootstrapConfig
+
+ // Set up a zero-valued tls.Config, we'll use this to tell if we need to do
+ // any TLS setup.
+ tlsconfig := tls.Config{}
+
  bootstrap := app.Command("bootstrap", "Generate bootstrap configuration.")
  path := bootstrap.Arg("path", "Configuration file.").Required().String()
  bootstrap.Flag("admin-address", "Envoy admin interface address").StringVar(&config.AdminAddress)
  bootstrap.Flag("admin-port", "Envoy admin interface port").IntVar(&config.AdminPort)
  bootstrap.Flag("xds-address", "xDS gRPC API address").StringVar(&config.XDSAddress)
  bootstrap.Flag("xds-port", "xDS gRPC API port").IntVar(&config.XDSGRPCPort)
+ bootstrap.Flag("envoy-cafile", "gRPC CA Filename for Envoy to load").StringVar(&config.GrpcCABundle)
+ bootstrap.Flag("envoy-cert-file", "gRPC Client cert filename for Envoy to load").StringVar(&config.GrpcClientCert)
+ bootstrap.Flag("envoy-key-file", "gRPC Client key filename for Envoy to load").StringVar(&config.GrpcClientKey)
 
  // Get the running namespace passed via ENV var from the Kubernetes Downward API
  config.Namespace = getEnv("CONTOUR_NAMESPACE", "heptio-contour")
 
  cli := app.Command("cli", "A CLI client for the Heptio Contour Kubernetes ingress controller.")
  var client Client
  cli.Flag("contour", "contour host:port.").Default("127.0.0.1:8001").StringVar(&client.ContourAddr)
+ cli.Flag("cafile", "CA bundle file for connecting to a TLS-secured Contour").StringVar(&client.CAFile)
+ cli.Flag("cert-file", "Client certificate file for connecting to a TLS-secured Contour").StringVar(&client.ClientCert)
+ cli.Flag("key-file", "Client key file for connecting to a TLS-secured Contour").StringVar(&client.ClientKey)
 
  var resources []string
  cds := cli.Command("cds", "watch services.")
@@ -82,6 +97,9 @@ func main() {
  xdsPort := serve.Flag("xds-port", "xDS gRPC API port").Default("8001").Int()
  statsAddress := serve.Flag("stats-address", "Envoy /stats interface address").Default("0.0.0.0").String()
  statsPort := serve.Flag("stats-port", "Envoy /stats interface port").Default("8002").Int()
+ caFile := serve.Flag("contour-cafile", "CA bundle file name for serving gRPC with TLS").String()
+ clientCert := serve.Flag("contour-cert-file", "Contour certificate file name for serving gRPC over TLS").String()
+ clientKey := serve.Flag("contour-key-file", "Contour key file name for serving gRPC over TLS").String()
 
  ch := contour.CacheHandler{
  FieldLogger: log.WithField("context", "CacheHandler"),
@@ -166,6 +184,14 @@ func main() {
  stream := client.RouteStream()
  watchstream(stream, cache.SecretType, resources)
  case serve.FullCommand():
+ if *caFile != "" || *clientCert != "" || *clientKey != "" {
+ // If one of the three TLS commands is not empty, they all must be not empty
+ if !(*caFile != "" && *clientCert != "" && *clientKey != "") {
+ log.Fatal("Can't have some TLS config, it's all or none")
+ }
+ setupTLSConfig(&tlsconfig, *caFile, *clientCert, *clientKey, log)
+ }
+
  log.Infof("args: %v", args)
  var g workgroup.Group
 
@@ -228,9 +254,20 @@ func main() {
  g.Add(func(stop <-chan struct{}) error {
  log := log.WithField("context", "grpc")
  addr := net.JoinHostPort(*xdsAddr, strconv.Itoa(*xdsPort))
- l, err := net.Listen("tcp", addr)
- if err != nil {
- return err
+
+ var l net.Listener
+ var err error
+ if tlsconfig.ClientAuth != tls.NoClientCert {
+ log.Info("Setting up TLS for gRPC")
+ l, err = tls.Listen("tcp", addr, &tlsconfig)
+ if err != nil {
+ return err
+ }
+ } else {
+ l, err = net.Listen("tcp", addr)
+ if err != nil {
+ return err
+ }
  }
 
  s := grpc.NewAPI(log, map[string]grpc.Resource{
@@ -312,3 +349,30 @@ func getEnv(key, fallback string) string {
  }
  return value
 }
+
+func setupTLSConfig(config *tls.Config, caFile string, clientCert string, clientKey string, log *logrus.Logger) error {
+
+ cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
+ if err != nil {
+ return err
+ }
+
+ ca, err := ioutil.ReadFile(caFile)
+
+ if err != nil {
+ return err
+ }
+ certPool := x509.NewCertPool()
+
+ if ok := certPool.AppendCertsFromPEM(ca); !ok {
+ return err
+ }
+
+ config.Certificates = []tls.Certificate{cert}
+ config.ClientAuth = tls.RequireAndVerifyClientCert
+ config.ClientCAs = certPool
+ config.Rand = rand.Reader
+
+ return nil
+
+}

internal/envoy/bootstrap.go

@@ -14,11 +14,13 @@
 package envoy
 
 import (
+ "log"
  "strconv"
  "strings"
  "time"
 
  api "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+ "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
  clusterv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2/cluster"
  "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
  "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
@@ -85,9 +87,62 @@ func Bootstrap(c *BootstrapConfig) *bootstrap.Bootstrap {
  },
  }
 
+ if c.GrpcClientCert != "" || c.GrpcClientKey != "" {
+ // If one of the two TLS options is not empty, they all must be not empty
+ if !(c.GrpcClientCert != "" && c.GrpcClientKey != "") {
+ log.Fatal("Can't have some TLS config, it's all or none")
+ }
+ b.StaticResources.Clusters[0].TlsContext = grpcUpstreamTLSContext(c.GrpcCABundle, c.GrpcClientCert, c.GrpcClientKey)
+ }
+
  return b
 }
 
+func grpcUpstreamTLSContext(cafile string, certfile string, keyfile string) *auth.UpstreamTlsContext {
+ if certfile == "" {
+ // Nothig to do
+ return nil
+ }
+
+ if certfile == "" {
+ // Nothing to do
+ return nil
+ }
+
+ context := &auth.UpstreamTlsContext{
+ CommonTlsContext: &auth.CommonTlsContext{
+ TlsCertificates: []*auth.TlsCertificate{
+ {
+ CertificateChain: &core.DataSource{
+ Specifier: &core.DataSource_Filename{
+ Filename: certfile,
+ },
+ },
+ PrivateKey: &core.DataSource{
+ Specifier: &core.DataSource_Filename{
+ Filename: keyfile,
+ },
+ },
+ },
+ },
+ },
+ }
+
+ if cafile != "" {
+ context.CommonTlsContext.ValidationContextType = &auth.CommonTlsContext_ValidationContext{
+ ValidationContext: &auth.CertificateValidationContext{
+ TrustedCa: &core.DataSource{
+ Specifier: &core.DataSource_Filename{
+ Filename: cafile,
+ },
+ },
+ VerifySubjectAltName: []string{"contour"},
+ },
+ }
+ }
+ return context
+}
+
 func stringOrDefault(s, def string) string {
  if s == "" {
  return def
@@ -126,4 +181,15 @@ type BootstrapConfig struct {
 
  // Namespace is the namespace where Contour is running
  Namespace string
+
+ //GrpcCABundle is the filename that contains a CA certificate chain that can
+ //verify the client cert.
+ GrpcCABundle string
+
+ // GrpcClientCert is the filename that contains a client certificate. May contain a full bundle if you
+ // don't want to pass a CA Bundle.
+ GrpcClientCert string
+
+ // GrpcClientKey is the filename that contains a client key for secure gRPC with TLS.
+ GrpcClientKey string
 }