TLS cert provisioning

agent/acl_test.go

@@ -64,7 +64,7 @@ func NewTestACLAgent(name string, hcl string, resolveFn func(string) (acl.Author
 		config.Source{Name: a.Name + ".data_dir", Format: "hcl", Data: hclDataDir},
 	)
 
-	agent, err := New(a.Config)
+	agent, err := New(a.Config, nil)
 	if err != nil {
 		panic(fmt.Sprintf("Error creating agent: %v", err))
 	}

agent/agent.go

@@ -71,6 +71,12 @@ const (
 		"but no reason was provided. This is a default message."
 	defaultServiceMaintReason = "Maintenance mode is enabled for this " +
 		"service, but no reason was provided. This is a default message."
+
+	// ID of the roots watch
+	rootsWatchID = "roots"
+
+	// ID of the leaf watch
+	leafWatchID = "leaf"
 )
 
 type configSource int
@@ -202,6 +208,8 @@ type Agent struct {
 	shutdownCh   chan struct{}
 	shutdownLock sync.Mutex
 
+	InterruptStartCh chan struct{}
+
 	// joinLANNotifier is called after a successful JoinLAN.
 	joinLANNotifier notifier
 
@@ -264,40 +272,48 @@ type Agent struct {
 	persistedTokensLock sync.RWMutex
 }
 
-func New(c *config.RuntimeConfig) (*Agent, error) {
+func New(c *config.RuntimeConfig, logger *log.Logger) (*Agent, error) {
 	if c.Datacenter == "" {
 		return nil, fmt.Errorf("Must configure a Datacenter")
 	}
 	if c.DataDir == "" && !c.DevMode {
 		return nil, fmt.Errorf("Must configure a DataDir")
 	}
 
-	a := &Agent{
-		config:          c,
-		checkReapAfter:  make(map[types.CheckID]time.Duration),
-		checkMonitors:   make(map[types.CheckID]*checks.CheckMonitor),
-		checkTTLs:       make(map[types.CheckID]*checks.CheckTTL),
-		checkHTTPs:      make(map[types.CheckID]*checks.CheckHTTP),
-		checkTCPs:       make(map[types.CheckID]*checks.CheckTCP),
-		checkGRPCs:      make(map[types.CheckID]*checks.CheckGRPC),
-		checkDockers:    make(map[types.CheckID]*checks.CheckDocker),
-		checkAliases:    make(map[types.CheckID]*checks.CheckAlias),
-		eventCh:         make(chan serf.UserEvent, 1024),
-		eventBuf:        make([]*UserEvent, 256),
-		joinLANNotifier: &systemd.Notifier{},
-		reloadCh:        make(chan chan error),
-		retryJoinCh:     make(chan error),
-		shutdownCh:      make(chan struct{}),
-		endpoints:       make(map[string]string),
-		tokens:          new(token.Store),
-	}
-	a.serviceManager = NewServiceManager(a)
+	a := Agent{
+		config:           c,
+		checkReapAfter:   make(map[types.CheckID]time.Duration),
+		checkMonitors:    make(map[types.CheckID]*checks.CheckMonitor),
+		checkTTLs:        make(map[types.CheckID]*checks.CheckTTL),
+		checkHTTPs:       make(map[types.CheckID]*checks.CheckHTTP),
+		checkTCPs:        make(map[types.CheckID]*checks.CheckTCP),
+		checkGRPCs:       make(map[types.CheckID]*checks.CheckGRPC),
+		checkDockers:     make(map[types.CheckID]*checks.CheckDocker),
+		checkAliases:     make(map[types.CheckID]*checks.CheckAlias),
+		eventCh:          make(chan serf.UserEvent, 1024),
+		eventBuf:         make([]*UserEvent, 256),
+		joinLANNotifier:  &systemd.Notifier{},
+		reloadCh:         make(chan chan error),
+		retryJoinCh:      make(chan error),
+		shutdownCh:       make(chan struct{}),
+		InterruptStartCh: make(chan struct{}),
+		endpoints:        make(map[string]string),
+		tokens:           new(token.Store),
+		logger:           logger,
+	}
+	a.serviceManager = NewServiceManager(&a)
 
 	if err := a.initializeACLs(); err != nil {
 		return nil, err
 	}
 
-	return a, nil
+	// Retrieve or generate the node ID before setting up the rest of the
+	// agent, which depends on it.
+	if err := a.setupNodeID(c); err != nil {
+		return nil, fmt.Errorf("Failed to setup node ID: %v", err)
+	}
+
+	return &a, nil
 }
 
 func LocalConfig(cfg *config.RuntimeConfig) local.Config {
@@ -348,20 +364,6 @@ func (a *Agent) Start() error {
 
 	c := a.config
 
-	logOutput := a.LogOutput
-	if a.logger == nil {
-		if logOutput == nil {
-			logOutput = os.Stderr
-		}
-		a.logger = log.New(logOutput, "", log.LstdFlags)
-	}
-
-	// Retrieve or generate the node ID before setting up the rest of the
-	// agent, which depends on it.
-	if err := a.setupNodeID(c); err != nil {
-		return fmt.Errorf("Failed to setup node ID: %v", err)
-	}
-
 	// Warn if the node name is incompatible with DNS
 	if InvalidDnsRe.MatchString(a.config.NodeName) {
 		a.logger.Printf("[WARN] agent: Node name %q will not be discoverable "+
@@ -433,6 +435,21 @@ func (a *Agent) Start() error {
 	// populated from above.
 	a.registerCache()
 
+	if a.config.AutoEncryptTLS && !a.config.ServerMode {
+		reply, err := a.setupClientAutoEncrypt()
+		if err != nil {
+			return fmt.Errorf("AutoEncrypt failed: %s", err)
+		}
+		rootsReq, leafReq, err := a.setupClientAutoEncryptCache(reply)
+		if err != nil {
+			return fmt.Errorf("AutoEncrypt failed: %s", err)
+		}
+		if err = a.setupClientAutoEncryptWatching(rootsReq, leafReq); err != nil {
+			return fmt.Errorf("AutoEncrypt failed: %s", err)
+		}
+		a.logger.Printf("[INFO] AutoEncrypt: upgraded to TLS")
+	}
+
 	// Load checks/services/metadata.
 	if err := a.loadServices(c); err != nil {
 		return err
@@ -532,6 +549,158 @@ func (a *Agent) Start() error {
 	return nil
 }
 
+func (a *Agent) setupClientAutoEncrypt() (*structs.SignedResponse, error) {
+	client := a.delegate.(*consul.Client)
+
+	addrs := a.config.StartJoinAddrsLAN
+	disco, err := newDiscover()
+	if err != nil && len(addrs) == 0 {
+		return nil, err
+	}
+	addrs = append(addrs, retryJoinAddrs(disco, "LAN", a.config.RetryJoinLAN, a.logger)...)
+
+	reply, priv, err := client.RequestAutoEncryptCerts(addrs, a.config.ServerPort, a.tokens.AgentToken(), a.InterruptStartCh)
+	if err != nil {
+		return nil, err
+	}
+
+	connectCAPems := []string{}
+	for _, ca := range reply.ConnectCARoots.Roots {
+		connectCAPems = append(connectCAPems, ca.RootCert)
+	}
+	if err := a.tlsConfigurator.UpdateAutoEncrypt(reply.ManualCARoots, connectCAPems, reply.IssuedCert.CertPEM, priv, reply.VerifyServerHostname); err != nil {
+		return nil, err
+	}
+	return reply, nil
+
+}
+
+func (a *Agent) setupClientAutoEncryptCache(reply *structs.SignedResponse) (*structs.DCSpecificRequest, *cachetype.ConnectCALeafRequest, error) {
+	rootsReq := &structs.DCSpecificRequest{
+		Datacenter:   a.config.Datacenter,
+		QueryOptions: structs.QueryOptions{Token: a.tokens.AgentToken()},
+	}
+
+	// prepolutate roots cache
+	rootRes := cache.FetchResult{Value: &reply.ConnectCARoots, Index: reply.ConnectCARoots.QueryMeta.Index}
+	if err := a.cache.Prepopulate(cachetype.ConnectCARootName, rootRes, a.config.Datacenter, a.tokens.AgentToken(), rootsReq.CacheInfo().Key); err != nil {
+		return nil, nil, err
+	}
+
+	leafReq := &cachetype.ConnectCALeafRequest{
+		Datacenter: a.config.Datacenter,
+		Token:      a.tokens.AgentToken(),
+		Agent:      a.config.NodeName,
+	}
+
+	// prepolutate leaf cache
+	certRes := cache.FetchResult{Value: &reply.IssuedCert, Index: reply.ConnectCARoots.QueryMeta.Index}
+	if err := a.cache.Prepopulate(cachetype.ConnectCALeafName, certRes, a.config.Datacenter, a.tokens.AgentToken(), leafReq.Key()); err != nil {
+		return nil, nil, err
+	}
+	return rootsReq, leafReq, nil
+}
+
+func (a *Agent) setupClientAutoEncryptWatching(rootsReq *structs.DCSpecificRequest, leafReq *cachetype.ConnectCALeafRequest) error {
+	// setup watches
+	ch := make(chan cache.UpdateEvent, 10)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Watch for root changes
+	err := a.cache.Notify(ctx, cachetype.ConnectCARootName, rootsReq, rootsWatchID, ch)
+	if err != nil {
+		cancel()
+		return err
+	}
+
+	// Watch the leaf cert
+	err = a.cache.Notify(ctx, cachetype.ConnectCALeafName, leafReq, leafWatchID, ch)
+	if err != nil {
+		cancel()
+		return err
+	}
+
+	// Setup actions in case the watches are firing.
+	go func() {
+		for {
+			select {
+			case <-a.shutdownCh:
+				cancel()
+				return
+			case <-ctx.Done():
+				return
+			case u := <-ch:
+				switch u.CorrelationID {
+				case rootsWatchID:
+					roots, ok := u.Result.(*structs.IndexedCARoots)
+					if !ok {
+						err := fmt.Errorf("invalid type for roots response: %T", u.Result)
+						a.logger.Printf("[ERR] %s watch error: %s", u.CorrelationID, err)
+						continue
+					}
+					pems := []string{}
+					for _, root := range roots.Roots {
+						pems = append(pems, root.RootCert)
+					}
+					a.tlsConfigurator.UpdateAutoEncryptCA(pems)
+				case leafWatchID:
+					leaf, ok := u.Result.(*structs.IssuedCert)
+					if !ok {
+						err := fmt.Errorf("invalid type for leaf response: %T", u.Result)
+						a.logger.Printf("[ERR] %s watch error: %s", u.CorrelationID, err)
+						continue
+					}
+					a.tlsConfigurator.UpdateAutoEncryptCert(leaf.CertPEM, leaf.PrivateKeyPEM)
+				}
+			}
+		}
+	}()
+
+	// Setup safety net in case the auto_encrypt cert doesn't get renewed
+	// in time. The agent would be stuck in that case because the watches
+	// never use the AutoEncrypt.Sign endpoint.
+	go func() {
+		for {
+
+			// Check 10sec after cert expires. The agent cache
+			// should be handling the expiration and renew before
+			// it.
+			// If there is no cert, AutoEncryptCertNotAfter returns
+			// a value in the past which immediately triggers the
+			// renew, but this case shouldn't happen because at
+			// this point, auto_encrypt was just being setup
+			// successfully.
+			interval := a.tlsConfigurator.AutoEncryptCertNotAfter().Sub(time.Now().Add(10 * time.Second))
+			a.logger.Printf("[DEBUG] AutoEncrypt: client certificate expiration check in %s", interval)
+			select {
+			case <-a.shutdownCh:
+				return
+			case <-time.After(interval):
+				// check auto encrypt client cert expiration
+				if a.tlsConfigurator.AutoEncryptCertExpired() {
+					a.logger.Printf("[DEBUG] AutoEncrypt: client certificate expired.")
+					reply, err := a.setupClientAutoEncrypt()
+					if err != nil {
+						a.logger.Printf("[ERR] AutoEncrypt: client certificate expired, failed to renew: %s", err)
+						// in case of an error, try again in one minute
+						interval = time.Minute
+						continue
+					}
+					_, _, err = a.setupClientAutoEncryptCache(reply)
+					if err != nil {
+						a.logger.Printf("[ERR] AutoEncrypt: client certificate expired, failed to populate cache: %s", err)
+						// in case of an error, try again in one minute
+						interval = time.Minute
+						continue
+					}
+				}
+			}
+		}
+	}()
+
+	return nil
+}
+
 func (a *Agent) listenAndServeGRPC() error {
 	if len(a.config.GRPCAddrs) < 1 {
 		return nil
@@ -1088,6 +1257,8 @@ func (a *Agent) consulConfig() (*consul.Config, error) {
 	base.TLSCipherSuites = a.config.TLSCipherSuites
 	base.TLSPreferServerCipherSuites = a.config.TLSPreferServerCipherSuites
 
+	base.AutoEncryptAllowTLS = a.config.AutoEncryptAllowTLS
+
 	// Copy the Connect CA bootstrap config
 	if a.config.ConnectEnabled {
 		base.ConnectEnabled = true

agent/agent_test.go

@@ -3871,3 +3871,20 @@ func TestAgent_ReloadConfigTLSConfigFailure(t *testing.T) {
 	require.Len(t, tlsConf.ClientCAs.Subjects(), 1)
 	require.Len(t, tlsConf.RootCAs.Subjects(), 1)
 }
+
+func TestAgent_consulConfig(t *testing.T) {
+	t.Parallel()
+	dataDir := testutil.TempDir(t, "agent") // we manage the data dir
+	defer os.RemoveAll(dataDir)
+	hcl := `
+		data_dir = "` + dataDir + `"
+		verify_incoming = true
+		ca_file = "../test/ca/root.cer"
+		cert_file = "../test/key/ourdomain.cer"
+		key_file = "../test/key/ourdomain.key"
+		auto_encrypt { allow_tls = true }
+	`
+	a := NewTestAgent(t, t.Name(), hcl)
+	defer a.Shutdown()
+	require.True(t, a.consulConfig().AutoEncryptAllowTLS)
+}

agent/cache-types/connect_ca_leaf.go

@@ -501,12 +501,23 @@ func (c *ConnectCALeaf) generateNewLeaf(req *ConnectCALeafRequest,
 		return result, errors.New("cluster has no CA bootstrapped yet")
 	}
 
-	// Build the service ID
-	serviceID := &connect.SpiffeIDService{
-		Host:       roots.TrustDomain,
-		Datacenter: req.Datacenter,
-		Namespace:  "default",
-		Service:    req.Service,
+	// Build the cert uri
+	var id connect.CertURI
+	if req.Service != "" {
+		id = &connect.SpiffeIDService{
+			Host:       roots.TrustDomain,
+			Datacenter: req.Datacenter,
+			Namespace:  "default",
+			Service:    req.Service,
+		}
+	} else if req.Agent != "" {
+		id = &connect.SpiffeIDAgent{
+			Host:       roots.TrustDomain,
+			Datacenter: req.Datacenter,
+			Agent:      req.Agent,
+		}
+	} else {
+		return result, errors.New("URI must be either service or agent")
 	}
 
 	// Create a new private key
@@ -516,7 +527,7 @@ func (c *ConnectCALeaf) generateNewLeaf(req *ConnectCALeafRequest,
 	}
 
 	// Create a CSR.
-	csr, err := connect.CreateCSR(serviceID, pk)
+	csr, err := connect.CreateCSR(id, pk)
 	if err != nil {
 		return result, err
 	}
@@ -606,14 +617,23 @@ type ConnectCALeafRequest struct {
 	Token         string
 	Datacenter    string
 	Service       string // Service name, not ID
+	Agent         string // Agent name, not ID
 	MinQueryIndex uint64
 	MaxQueryTime  time.Duration
 }
 
+func (r *ConnectCALeafRequest) Key() string {
+	if len(r.Agent) > 0 {
+		return fmt.Sprintf("agent:%s", r.Agent)
+	}
+
+	return fmt.Sprintf("service:%s", r.Service)
+}
+
 func (r *ConnectCALeafRequest) CacheInfo() cache.RequestInfo {
 	return cache.RequestInfo{
 		Token:      r.Token,
-		Key:        r.Service,
+		Key:        r.Key(),
 		Datacenter: r.Datacenter,
 		MinIndex:   r.MinQueryIndex,
 		Timeout:    r.MaxQueryTime,