From ddca80dbc09590031fa65e7c9e476654c2284db8 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Thu, 19 Dec 2019 16:30:59 -0800 Subject: [PATCH] Support TLS-enabled servers for the server-acl-init - add new flags, -use-https, -consul-ca-cert, and -consul-tls-server-name. - Bump Consul version to 1.6.2 to use tlsutil certificate generation functions in tests. --- go.sum | 2 + subcommand/server-acl-init/command.go | 57 ++++++--- subcommand/server-acl-init/command_test.go | 131 ++++++++++++++++++++- 3 files changed, 173 insertions(+), 17 deletions(-) diff --git a/go.sum b/go.sum index acbd9f3060..feecfb3399 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,7 @@ cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/azure-sdk-for-go v16.0.0+incompatible h1:gr1qKY/Ll72VjFTZmaBwRK1yQHAxCnV25ekOKroc9ws= github.com/Azure/azure-sdk-for-go v16.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v10.7.0+incompatible h1:dB+dKSLGdJLEhU/FoZTSNSPMZuE5H4M5p5zgSct7qwM= github.com/Azure/go-autorest v10.7.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v10.15.3+incompatible h1:nhKI/bvazIs3C3TFGoSqKY6hZ8f5od5mb5/UcS6HVIY= github.com/Azure/go-autorest v10.15.3+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -436,6 +437,7 @@ gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNat gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/subcommand/server-acl-init/command.go b/subcommand/server-acl-init/command.go index be098769ae..0a29ab2046 100644 --- a/subcommand/server-acl-init/command.go +++ b/subcommand/server-acl-init/command.go @@ -42,6 +42,9 @@ type Command struct { flagCreateMeshGatewayToken bool flagLogLevel string flagTimeout string + flagConsulCACert string + flagConsulTLSServerName string + flagUseHTTPS bool clientset kubernetes.Interface // cmdTimeout is cancelled when the command timeout is reached. @@ -82,6 +85,12 @@ func (c *Command) init() { "Toggle for creating a token for a Connect mesh gateway") c.flags.StringVar(&c.flagTimeout, "timeout", "10m", "How long we'll try to bootstrap ACLs for before timing out, e.g. 1ms, 2s, 3m") + c.flags.StringVar(&c.flagConsulCACert, "consul-ca-cert", "", + "Path to the PEM-encoded CA certificate of the Consul cluster.") + c.flags.StringVar(&c.flagConsulTLSServerName, "consul-tls-server-name", "", + "The server name to set as the SNI header when sending HTTPS requests to Consul.") + c.flags.BoolVar(&c.flagUseHTTPS, "use-https", false, + "Toggle for using HTTPS for all API calls to Consul.") c.flags.StringVar(&c.flagLogLevel, "log-level", "info", "Log verbosity level. Supported values (in order of detail) are \"trace\", "+ "\"debug\", \"info\", \"warn\", and \"error\".") @@ -162,6 +171,10 @@ func (c *Command) Run(args []string) int { } } + scheme := "http" + if c.flagUseHTTPS { + scheme = "https" + } // Wait if there's a rollout of servers. ssName := c.withPrefix("server") err = c.untilSucceeds(fmt.Sprintf("waiting for rollout of statefulset %s", ssName), func() error { @@ -196,7 +209,7 @@ func (c *Command) Run(args []string) int { logger.Info(fmt.Sprintf("ACLs already bootstrapped - retrieved bootstrap token from Secret %q", bootTokenSecretName)) } else { logger.Info("No bootstrap token from previous installation found, continuing on to bootstrapping") - bootstrapToken, err = c.bootstrapServers(logger, bootTokenSecretName) + bootstrapToken, err = c.bootstrapServers(logger, bootTokenSecretName, scheme) if err != nil { logger.Error(err.Error()) return 1 @@ -204,7 +217,7 @@ func (c *Command) Run(args []string) int { } // For all of the next operations we'll need a Consul client. - serverPods, err := c.getConsulServers(logger, 1) + serverPods, err := c.getConsulServers(logger, 1, scheme) if err != nil { logger.Error(err.Error()) return 1 @@ -212,8 +225,12 @@ func (c *Command) Run(args []string) int { serverAddr := serverPods[0].Addr consulClient, err := api.NewClient(&api.Config{ Address: serverAddr, - Scheme: "http", - Token: string(bootstrapToken), + Scheme: scheme, + Token: bootstrapToken, + TLSConfig: api.TLSConfig{ + Address: c.flagConsulTLSServerName, + CAFile: c.flagConsulCACert, + }, }) if err != nil { logger.Error(fmt.Sprintf("Error creating Consul client for addr %q: %s", serverAddr, err)) @@ -312,7 +329,7 @@ func (c *Command) configureKubeClient() error { // getConsulServers returns n Consul server pods with their http addresses. // If there are less server pods than 'n' then the function will wait. -func (c *Command) getConsulServers(logger hclog.Logger, n int) ([]podAddr, error) { +func (c *Command) getConsulServers(logger hclog.Logger, n int, scheme string) ([]podAddr, error) { var serverPods *apiv1.PodList err := c.untilSucceeds("discovering Consul server pods", func() error { @@ -345,12 +362,12 @@ func (c *Command) getConsulServers(logger hclog.Logger, n int) ([]podAddr, error for _, pod := range serverPods.Items { var httpPort int32 for _, p := range pod.Spec.Containers[0].Ports { - if p.Name == "http" { + if p.Name == scheme { httpPort = p.ContainerPort } } if httpPort == 0 { - return nil, fmt.Errorf("pod %s has no port labeled 'http'", pod.Name) + return nil, fmt.Errorf("pod %s has no port labeled '%s'", pod.Name, scheme) } addr := fmt.Sprintf("%s:%d", pod.Status.PodIP, httpPort) podAddrs = append(podAddrs, podAddr{ @@ -362,8 +379,8 @@ func (c *Command) getConsulServers(logger hclog.Logger, n int) ([]podAddr, error } // bootstrapServers bootstraps ACLs and ensures each server has an ACL token. -func (c *Command) bootstrapServers(logger hclog.Logger, bootTokenSecretName string) (string, error) { - serverPods, err := c.getConsulServers(logger, c.flagReplicas) +func (c *Command) bootstrapServers(logger hclog.Logger, bootTokenSecretName, scheme string) (string, error) { + serverPods, err := c.getConsulServers(logger, c.flagReplicas, scheme) if err != nil { return "", err } @@ -373,7 +390,11 @@ func (c *Command) bootstrapServers(logger hclog.Logger, bootTokenSecretName stri firstServerAddr := serverPods[0].Addr consulClient, err := api.NewClient(&api.Config{ Address: firstServerAddr, - Scheme: "http", + Scheme: scheme, + TLSConfig: api.TLSConfig{ + Address: c.flagConsulTLSServerName, + CAFile: c.flagConsulCACert, + }, }) if err != nil { return "", fmt.Errorf("creating Consul client for address %s: %s", firstServerAddr, err) @@ -434,15 +455,19 @@ func (c *Command) bootstrapServers(logger hclog.Logger, bootTokenSecretName stri // set. consulClient, err = api.NewClient(&api.Config{ Address: firstServerAddr, - Scheme: "http", + Scheme: scheme, Token: string(bootstrapToken), + TLSConfig: api.TLSConfig{ + Address: c.flagConsulTLSServerName, + CAFile: c.flagConsulCACert, + }, }) if err != nil { return "", fmt.Errorf("creating Consul client for address %s: %s", firstServerAddr, err) } // Create new tokens for each server and apply them. - if err := c.setServerTokens(logger, consulClient, serverPods, string(bootstrapToken)); err != nil { + if err := c.setServerTokens(logger, consulClient, serverPods, string(bootstrapToken), scheme); err != nil { return "", err } return string(bootstrapToken), nil @@ -451,7 +476,7 @@ func (c *Command) bootstrapServers(logger hclog.Logger, bootTokenSecretName stri // setServerTokens creates policies and associated ACL token for each server // and then provides the token to the server. func (c *Command) setServerTokens(logger hclog.Logger, consulClient *api.Client, - serverPods []podAddr, bootstrapToken string) error { + serverPods []podAddr, bootstrapToken, scheme string) error { // Create agent policy. agentPolicy := api.ACLPolicy{ Name: "agent-token", @@ -497,8 +522,12 @@ func (c *Command) setServerTokens(logger hclog.Logger, consulClient *api.Client, // server specifically. serverClient, err := api.NewClient(&api.Config{ Address: pod.Addr, - Scheme: "http", + Scheme: scheme, Token: bootstrapToken, + TLSConfig: api.TLSConfig{ + Address: c.flagConsulTLSServerName, + CAFile: c.flagConsulCACert, + }, }) if err != nil { return fmt.Errorf(" creating Consul client for address %q: %s", pod.Addr, err) diff --git a/subcommand/server-acl-init/command_test.go b/subcommand/server-acl-init/command_test.go index 4cfc744ae3..5b9fee4cad 100644 --- a/subcommand/server-acl-init/command_test.go +++ b/subcommand/server-acl-init/command_test.go @@ -1,20 +1,25 @@ package serveraclinit import ( + "crypto/x509" "encoding/base64" "fmt" "github.com/hashicorp/consul/agent" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/tlsutil" "github.com/mitchellh/cli" "github.com/stretchr/testify/require" + "io/ioutil" appv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "math/rand" + "net" "net/http" "net/http/httptest" "net/url" + "os" "strconv" "testing" "time" @@ -1146,9 +1151,65 @@ func TestRun_Timeout(t *testing.T) { require.Equal(1, responseCode, ui.ErrorWriter.String()) } +// Test that the bootstrapping process can make calls to Consul API over HTTPS +// when the consul agent is configured with HTTPS only (HTTP disabled). +func TestRun_HTTPS(t *testing.T) { + t.Parallel() + require := require.New(t) + k8s := fake.NewSimpleClientset() + + caFile, certFile, keyFile, cleanup := generateServerCerts(t) + defer cleanup() + + agentConfig := fmt.Sprintf(` + primary_datacenter = "dc1" + acl { + enabled = true + } + ca_file = "%s" + cert_file = "%s" + key_file = "%s"`, caFile, certFile, keyFile) + + a := &agent.TestAgent{ + Name: t.Name(), + HCL: agentConfig, + UseTLS: true, // this also disables HTTP port + } + + a.Start() + defer a.Shutdown() + + createTestK8SResources(t, k8s, a, resourcePrefix, "https") + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + responseCode := cmd.Run([]string{ + "-server-label-selector=component=server,app=consul,release=" + releaseName, + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + ns, + "-use-https", + "-consul-tls-server-name", "server.dc1.consul", + "-consul-ca-cert", caFile, + "-expected-replicas=1", + }) + require.Equal(0, responseCode, ui.ErrorWriter.String()) + + // Test that the bootstrap token is created to make sure the bootstrapping succeeded. + // The presence of the bootstrap token tells us that the API calls to Consul have been successful. + tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(resourcePrefix+"-bootstrap-acl-token", metav1.GetOptions{}) + require.NoError(err) + require.NotNil(tokenSecret) + _, ok := tokenSecret.Data["token"] + require.True(ok) +} + // Set up test consul agent and kubernetes cluster. func completeSetup(t *testing.T, prefix string) (*fake.Clientset, *agent.TestAgent) { - require := require.New(t) k8s := fake.NewSimpleClientset() a := agent.NewTestAgent(t, t.Name(), ` @@ -1157,6 +1218,14 @@ func completeSetup(t *testing.T, prefix string) (*fake.Clientset, *agent.TestAge enabled = true }`) + createTestK8SResources(t, k8s, a, prefix, "http") + + return k8s, a +} + +// Create test k8s resources (server pods and server stateful set) +func createTestK8SResources(t *testing.T, k8s *fake.Clientset, a *agent.TestAgent, prefix, scheme string) { + require := require.New(t) consulURL, err := url.Parse("http://" + a.HTTPAddr()) require.NoError(err) port, err := strconv.Atoi(consulURL.Port()) @@ -1181,7 +1250,7 @@ func completeSetup(t *testing.T, prefix string) (*fake.Clientset, *agent.TestAge Name: "consul", Ports: []v1.ContainerPort{ { - Name: "http", + Name: scheme, ContainerPort: int32(port), }, }, @@ -1207,7 +1276,6 @@ func completeSetup(t *testing.T, prefix string) (*fake.Clientset, *agent.TestAge }, }) require.NoError(err) - return k8s, a } // getBootToken gets the bootstrap token from the Kubernetes secret. It will @@ -1221,5 +1289,62 @@ func getBootToken(t *testing.T, k8s *fake.Clientset, prefix string) string { return string(bootToken) } +// generateServerCerts generates Consul CA +// and a server certificate and saves them to temp files. +// It returns file names in this order: +// CA certificate, server certificate, and server key. +// Note that it's the responsibility of the caller to +// remove the temporary files created by this function. +func generateServerCerts(t *testing.T) (string, string, string, func()) { + require := require.New(t) + + caFile, err := ioutil.TempFile("", "ca") + require.NoError(err) + + certFile, err := ioutil.TempFile("", "cert") + require.NoError(err) + + certKeyFile, err := ioutil.TempFile("", "key") + require.NoError(err) + + // Generate CA + sn, err := tlsutil.GenerateSerialNumber() + require.NoError(err) + + s, _, err := tlsutil.GeneratePrivateKey() + require.NoError(err) + + constraints := []string{"consul", "localhost"} + ca, err := tlsutil.GenerateCA(s, sn, 1, constraints) + require.NoError(err) + + // Generate Server Cert + name := fmt.Sprintf("server.%s.%s", "dc1", "consul") + DNSNames := []string{name, "localhost"} + IPAddresses := []net.IP{net.ParseIP("127.0.0.1")} + extKeyUsage := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} + + sn, err = tlsutil.GenerateSerialNumber() + require.NoError(err) + + pub, priv, err := tlsutil.GenerateCert(s, ca, sn, name, 1, DNSNames, IPAddresses, extKeyUsage) + require.NoError(err) + + // Write certs and key to files + _, err = caFile.WriteString(ca) + require.NoError(err) + _, err = certFile.WriteString(pub) + require.NoError(err) + _, err = certKeyFile.WriteString(priv) + require.NoError(err) + + cleanupFunc := func() { + os.Remove(caFile.Name()) + os.Remove(certFile.Name()) + os.Remove(certKeyFile.Name()) + } + return caFile.Name(), certFile.Name(), certKeyFile.Name(), cleanupFunc +} + var serviceAccountCACert = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURDekNDQWZPZ0F3SUJBZ0lRS3pzN05qbDlIczZYYzhFWG91MjVoekFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlRMU9XVTJaR00wTVMweU1EaG1MVFF3T1RVdFlUSTRPUzB4Wm1NM01EQmhZekZqWXpndwpIaGNOTVRrd05qQTNNVEF4TnpNeFdoY05NalF3TmpBMU1URXhOek14V2pBdk1TMHdLd1lEVlFRREV5UTFPV1UyClpHTTBNUzB5TURobUxUUXdPVFV0WVRJNE9TMHhabU0zTURCaFl6RmpZemd3Z2dFaU1BMEdDU3FHU0liM0RRRUIKQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURaakh6d3FvZnpUcEdwYzBNZElDUzdldXZmdWpVS0UzUEMvYXBmREFnQgo0anpFRktBNzgvOStLVUd3L2MvMFNIZVNRaE4rYThnd2xIUm5BejFOSmNmT0lYeTRkd2VVdU9rQWlGeEg4cGh0CkVDd2tlTk83ejhEb1Y4Y2VtaW5DUkhHamFSbW9NeHBaN2cycFpBSk5aZVB4aTN5MWFOa0ZBWGU5Z1NVU2RqUloKUlhZa2E3d2gyQU85azJkbEdGQVlCK3Qzdld3SjZ0d2pHMFR0S1FyaFlNOU9kMS9vTjBFMDFMekJjWnV4a04xawo4Z2ZJSHk3Yk9GQ0JNMldURURXLzBhQXZjQVByTzhETHFESis2TWpjM3I3K3psemw4YVFzcGIwUzA4cFZ6a2k1CkR6Ly84M2t5dTBwaEp1aWo1ZUI4OFY3VWZQWHhYRi9FdFY2ZnZyTDdNTjRmQWdNQkFBR2pJekFoTUE0R0ExVWQKRHdFQi93UUVBd0lDQkRBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCdgpRc2FHNnFsY2FSa3RKMHpHaHh4SjUyTm5SVjJHY0lZUGVOM1p2MlZYZTNNTDNWZDZHMzJQVjdsSU9oangzS21BCi91TWg2TmhxQnpzZWtrVHowUHVDM3dKeU0yT0dvblZRaXNGbHF4OXNGUTNmVTJtSUdYQ2Ezd0M4ZS9xUDhCSFMKdzcvVmVBN2x6bWozVFFSRS9XMFUwWkdlb0F4bjliNkp0VDBpTXVjWXZQMGhYS1RQQldsbnpJaWphbVU1MHIyWQo3aWEwNjVVZzJ4VU41RkxYL3Z4T0EzeTRyanBraldvVlFjdTFwOFRaclZvTTNkc0dGV3AxMGZETVJpQUhUdk9ICloyM2pHdWs2cm45RFVIQzJ4UGozd0NUbWQ4U0dFSm9WMzFub0pWNWRWZVE5MHd1c1h6M3ZURzdmaWNLbnZIRlMKeHRyNVBTd0gxRHVzWWZWYUdIMk8KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=" var serviceAccountToken = "ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklpSjkuZXlKcGMzTWlPaUpyZFdKbGNtNWxkR1Z6TDNObGNuWnBZMlZoWTJOdmRXNTBJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5dVlXMWxjM0JoWTJVaU9pSmtaV1poZFd4MElpd2lhM1ZpWlhKdVpYUmxjeTVwYnk5elpYSjJhV05sWVdOamIzVnVkQzl6WldOeVpYUXVibUZ0WlNJNkltdG9ZV3RwTFdGeVlXTm9ibWxrTFdOdmJuTjFiQzFqYjI1dVpXTjBMV2x1YW1WamRHOXlMV0YxZEdodFpYUm9iMlF0YzNaakxXRmpZMjlvYm1SaWRpSXNJbXQxWW1WeWJtVjBaWE11YVc4dmMyVnlkbWxqWldGalkyOTFiblF2YzJWeWRtbGpaUzFoWTJOdmRXNTBMbTVoYldVaU9pSnJhR0ZyYVMxaGNtRmphRzVwWkMxamIyNXpkV3d0WTI5dWJtVmpkQzFwYm1wbFkzUnZjaTFoZFhSb2JXVjBhRzlrTFhOMll5MWhZMk52ZFc1MElpd2lhM1ZpWlhKdVpYUmxjeTVwYnk5elpYSjJhV05sWVdOamIzVnVkQzl6WlhKMmFXTmxMV0ZqWTI5MWJuUXVkV2xrSWpvaU4yVTVOV1V4TWprdFpUUTNNeTB4TVdVNUxUaG1ZV0V0TkRJd01UQmhPREF3TVRJeUlpd2ljM1ZpSWpvaWMzbHpkR1Z0T25ObGNuWnBZMlZoWTJOdmRXNTBPbVJsWm1GMWJIUTZhMmhoYTJrdFlYSmhZMmh1YVdRdFkyOXVjM1ZzTFdOdmJtNWxZM1F0YVc1cVpXTjBiM0l0WVhWMGFHMWxkR2h2WkMxemRtTXRZV05qYjNWdWRDSjkuWWk2M01NdHpoNU1CV0tLZDNhN2R6Q0pqVElURTE1aWtGeV9UbnBka19Bd2R3QTlKNEFNU0dFZUhONXZXdEN1dUZqb19sTUpxQkJQSGtLMkFxYm5vRlVqOW01Q29wV3lxSUNKUWx2RU9QNGZVUS1SYzBXMVBfSmpVMXJaRVJIRzM5YjVUTUxnS1BRZ3V5aGFpWkVKNkNqVnRtOXdVVGFncmdpdXFZVjJpVXFMdUY2U1lObTZTckt0a1BTLWxxSU8tdTdDMDZ3Vms1bTV1cXdJVlFOcFpTSUNfNUxzNWFMbXlaVTNuSHZILVY3RTNIbUJoVnlaQUI3NmpnS0IwVHlWWDFJT3NrdDlQREZhck50VTNzdVp5Q2p2cUMtVUpBNnNZZXlTZTRkQk5Lc0tsU1o2WXV4VVVtbjFSZ3YzMllNZEltbnNXZzhraGYtekp2cWdXazdCNUVB"