Start work on rotation, updates #1860

Gopkg.toml

@@ -36,7 +36,7 @@ ignored = ["github.com/Sirupsen/logrus"]
 
 [[constraint]]
   name = "github.com/vulcand/predicate"
-  version = "v1.0.0"
+  version = "v1.1.0"
 
 [[constraint]]
   name = "github.com/docker/docker"

constants.go

@@ -234,6 +234,15 @@ const (
 
 	// Syslog is a mode for syslog logging
 	Syslog = "syslog"
+
+	// HumanDateFormat is a human readable date formatting
+	HumanDateFormat = "Jan _2 15:04 UTC"
+
+	// HumanDateFormatSeconds is a human readable date formatting with seconds
+	HumanDateFormatSeconds = "Jan _2 15:04:05 UTC"
+
+	// HumanDateFormatMilli is a human readable date formatting with milliseconds
+	HumanDateFormatMilli = "Jan _2 15:04:05.000 UTC"
 )
 
 // Component generates "component:subcomponent1:subcomponent2" strings used

lib/auth/apiserver.go

@@ -64,16 +64,14 @@ func NewAPIServer(config *APIConfig) http.Handler {
 
 	// Operations on certificate authorities
 	srv.GET("/:version/domain", srv.withAuth(srv.getDomainName))
+
 	srv.POST("/:version/authorities/:type", srv.withAuth(srv.upsertCertAuthority))
+	srv.POST("/:version/authorities/:type/rotate", srv.withAuth(srv.rotateCertAuthority))
+	srv.POST("/:version/authorities/:type/rotate/external", srv.withAuth(srv.rotateExternalCertAuthority))
 	srv.DELETE("/:version/authorities/:type/:domain", srv.withAuth(srv.deleteCertAuthority))
 	srv.GET("/:version/authorities/:type/:domain", srv.withAuth(srv.getCertAuthority))
 	srv.GET("/:version/authorities/:type", srv.withAuth(srv.getCertAuthorities))
 
-	// DELETE IN: 2.6.0
-	// Certificate exchange for cluster upgrades used to upgrade from 2.4.0
-	// to 2.5.0 clusters.
-	srv.POST("/:version/exchangecerts", srv.withAuth(srv.exchangeCerts))
-
 	// Generating certificates for user and host authorities
 	srv.POST("/:version/ca/host/certs", srv.withAuth(srv.generateHostCert))
 	srv.POST("/:version/ca/user/certs", srv.withAuth(srv.generateUserCert))
@@ -647,14 +645,6 @@ func (s *APIServer) authenticateSSHUser(auth ClientI, w http.ResponseWriter, r *
 	return auth.AuthenticateSSHUser(req)
 }
 
-func (s *APIServer) exchangeCerts(auth ClientI, w http.ResponseWriter, r *http.Request, p httprouter.Params, version string) (interface{}, error) {
-	var req ExchangeCertsRequest
-	if err := httplib.ReadJSON(r, &req); err != nil {
-		return nil, trace.Wrap(err)
-	}
-	return auth.ExchangeCerts(req)
-}
-
 func (s *APIServer) changePassword(auth ClientI, w http.ResponseWriter, r *http.Request, p httprouter.Params, version string) (interface{}, error) {
 	var req services.ChangePasswordReq
 	if err := httplib.ReadJSON(r, &req); err != nil {
@@ -931,6 +921,17 @@ func (s *APIServer) generateServerKeys(auth ClientI, w http.ResponseWriter, r *h
 	return keys, nil
 }
 
+func (s *APIServer) rotateCertAuthority(auth ClientI, w http.ResponseWriter, r *http.Request, p httprouter.Params, version string) (interface{}, error) {
+	var req RotateRequest
+	if err := httplib.ReadJSON(r, &req); err != nil {
+		return nil, trace.Wrap(err)
+	}
+	if err := auth.RotateCertAuthority(req); err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return message("ok"), nil
+}
+
 type upsertCertAuthorityRawReq struct {
 	CA  json.RawMessage `json:"ca"`
 	TTL time.Duration   `json:"ttl"`
@@ -954,6 +955,25 @@ func (s *APIServer) upsertCertAuthority(auth ClientI, w http.ResponseWriter, r *
 	return message("ok"), nil
 }
 
+type rotateExternalCertAuthorityRawReq struct {
+	CA json.RawMessage `json:"ca"`
+}
+
+func (s *APIServer) rotateExternalCertAuthority(auth ClientI, w http.ResponseWriter, r *http.Request, p httprouter.Params, version string) (interface{}, error) {
+	var req rotateExternalCertAuthorityRawReq
+	if err := httplib.ReadJSON(r, &req); err != nil {
+		return nil, trace.Wrap(err)
+	}
+	ca, err := services.GetCertAuthorityMarshaler().UnmarshalCertAuthority(req.CA)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	if err := auth.RotateExternalCertAuthority(ca); err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return message("ok"), nil
+}
+
 func (s *APIServer) getCertAuthorities(auth ClientI, w http.ResponseWriter, r *http.Request, p httprouter.Params, version string) (interface{}, error) {
 	loadKeys, _, err := httplib.ParseBool(r.URL.Query(), "load_keys")
 	if err != nil {

lib/auth/auth.go

@@ -24,8 +24,10 @@ limitations under the License.
 package auth
 
 import (
+	"context"
 	"crypto/x509"
 	"fmt"
+	"math/rand"
 	"net/url"
 	"sync"
 	"time"
@@ -78,6 +80,7 @@ func NewAuthServer(cfg *InitConfig, opts ...AuthServerOption) (*AuthServer, erro
 	if cfg.AuditLog == nil {
 		cfg.AuditLog = events.NewDiscardAuditLog()
 	}
+	closeCtx, cancelFunc := context.WithCancel(context.TODO())
 	as := AuthServer{
 		clusterName:          cfg.ClusterName,
 		bk:                   cfg.Backend,
@@ -93,13 +96,16 @@ func NewAuthServer(cfg *InitConfig, opts ...AuthServerOption) (*AuthServer, erro
 		oidcClients:          make(map[string]*oidcClient),
 		samlProviders:        make(map[string]*samlProvider),
 		githubClients:        make(map[string]*githubClient),
+		cancelFunc:           cancelFunc,
+		closeCtx:             closeCtx,
 	}
 	for _, o := range opts {
 		o(&as)
 	}
 	if as.clock == nil {
 		as.clock = clockwork.NewRealClock()
 	}
+	go as.runPeriodicOperations()
 
 	return &as, nil
 }
@@ -119,6 +125,9 @@ type AuthServer struct {
 	clock         clockwork.Clock
 	bk            backend.Backend
 
+	closeCtx   context.Context
+	cancelFunc context.CancelFunc
+
 	sshca.Authority
 
 	// AuthServiceName is a human-readable name of this CA. If several Auth services are running
@@ -137,7 +146,37 @@ type AuthServer struct {
 	clusterName services.ClusterName
 }
 
+// runPeriodicOperations runs some periodic bookkeeping operations
+// performed by auth server
+func (a *AuthServer) runPeriodicOperations() {
+	// run periodic functions with a semi-random period
+	// to avoid contention on the database in case if there are multiple
+	// auth servers running - so they don't compete trying
+	// to update the same resources.
+	r := rand.New(rand.NewSource(a.clock.Now().UnixNano()))
+	period := defaults.HighResPollingPeriod + time.Duration(r.Intn(int(defaults.HighResPollingPeriod/time.Second)))*time.Second
+	log.Debugf("Ticking with period: %v", period)
+	ticker := time.NewTicker(period)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-a.closeCtx.Done():
+			return
+		case <-ticker.C:
+			err := a.autoRotateCertAuthorities()
+			if err != nil {
+				if trace.IsCompareFailed(err) {
+					log.Debugf("Cert authority has been updated concurrently: %v", err)
+				} else {
+					log.Errorf("Failed to perform cert rotation check: %v", err)
+				}
+			}
+		}
+	}
+}
+
 func (a *AuthServer) Close() error {
+	a.cancelFunc()
 	if a.bk != nil {
 		return trace.Wrap(a.bk.Close())
 	}
@@ -303,15 +342,19 @@ func (s *AuthServer) generateUserCert(req certRequest) (*certs, error) {
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
-	hostCA, err := s.Trust.GetCertAuthority(services.CertAuthID{
+	// CHANGE IN (2.7.0) Use User CA and not host CA
+	// currently it is used for backwards compatibility
+	// as pre 2.6.0 remote clusters don't have TLS CAs stored
+	// for user certificate authorities.
+	userCA, err := s.Trust.GetCertAuthority(services.CertAuthID{
 		Type:       services.HostCA,
 		DomainName: clusterName,
 	}, true)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
 	// generate TLS certificate
-	tlsAuthority, err := hostCA.TLSCA()
+	tlsAuthority, err := userCA.TLSCA()
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -588,10 +631,18 @@ func (s *AuthServer) GenerateToken(req GenerateTokenRequest) (string, error) {
 // ClientCertPool returns trusted x509 cerificate authority pool
 func (s *AuthServer) ClientCertPool() (*x509.CertPool, error) {
 	pool := x509.NewCertPool()
-	authorities, err := s.GetCertAuthorities(services.HostCA, false)
+	var authorities []services.CertAuthority
+	hostCAs, err := s.GetCertAuthorities(services.HostCA, false)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
+	userCAs, err := s.GetCertAuthorities(services.UserCA, false)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	authorities = append(authorities, hostCAs...)
+	authorities = append(authorities, userCAs...)
+
 	for _, auth := range authorities {
 		for _, keyPair := range auth.GetTLSKeyPairs() {
 			cert, err := tlsca.ParseCertificatePEM(keyPair.Cert)

lib/auth/auth_with_roles.go

@@ -41,6 +41,10 @@ type AuthWithRoles struct {
 	alog       events.IAuditLog
 }
 
+func (a *AuthWithRoles) actionWithContext(ctx *services.Context, namespace string, resource string, action string) error {
+	return a.checker.CheckAccessToRule(ctx, namespace, resource, action)
+}
+
 func (a *AuthWithRoles) action(namespace string, resource string, action string) error {
 	return a.checker.CheckAccessToRule(&services.Context{User: a.user}, namespace, resource, action)
 }
@@ -91,16 +95,6 @@ func (a *AuthWithRoles) AuthenticateSSHUser(req AuthenticateSSHRequest) (*SSHLog
 	return a.authServer.AuthenticateSSHUser(req)
 }
 
-// ExchangeCerts exchanges TLS certificates for established host certificate authorities
-func (a *AuthWithRoles) ExchangeCerts(req ExchangeCertsRequest) (*ExchangeCertsResponse, error) {
-	// exchange request has it's own authentication, however this limits the requests
-	// types to proxies to make it harder to break
-	if !a.checker.HasRole(string(teleport.RoleProxy)) {
-		return nil, trace.AccessDenied("this request can be only executed by proxy")
-	}
-	return a.authServer.ExchangeCerts(req)
-}
-
 func (a *AuthWithRoles) GetSessions(namespace string) ([]session.Session, error) {
 	if err := a.action(namespace, services.KindSSHSession, services.VerbList); err != nil {
 		return nil, trace.Wrap(err)
@@ -134,16 +128,58 @@ func (a *AuthWithRoles) CreateCertAuthority(ca services.CertAuthority) error {
 	return trace.BadParameter("not implemented")
 }
 
-func (a *AuthWithRoles) UpsertCertAuthority(ca services.CertAuthority) error {
+// Rotate starts or restarts certificate rotation process
+func (a *AuthWithRoles) RotateCertAuthority(req RotateRequest) error {
+	if err := req.CheckAndSetDefaults(a.authServer.clock); err != nil {
+		return trace.Wrap(err)
+	}
 	if err := a.action(defaults.Namespace, services.KindCertAuthority, services.VerbCreate); err != nil {
 		return trace.Wrap(err)
 	}
 	if err := a.action(defaults.Namespace, services.KindCertAuthority, services.VerbUpdate); err != nil {
 		return trace.Wrap(err)
 	}
+	return a.authServer.RotateCertAuthority(req)
+}
+
+// RotateExternalCertAuthority rotates external certificate authority,
+// this method is called by remote trusted cluster and is used to update
+// only public keys and certificates of the certificate authority.
+func (a *AuthWithRoles) RotateExternalCertAuthority(ca services.CertAuthority) error {
+	if ca == nil {
+		return trace.BadParameter("missing certificate authority")
+	}
+	ctx := &services.Context{User: a.user, Resource: ca}
+	if err := a.actionWithContext(ctx, defaults.Namespace, services.KindCertAuthority, services.VerbRotate); err != nil {
+		return trace.Wrap(err)
+	}
+	return a.authServer.RotateExternalCertAuthority(ca)
+}
+
+func (a *AuthWithRoles) UpsertCertAuthority(ca services.CertAuthority) error {
+	if ca == nil {
+		return trace.BadParameter("missing certificate authority")
+	}
+	ctx := &services.Context{User: a.user, Resource: ca}
+	if err := a.actionWithContext(ctx, defaults.Namespace, services.KindCertAuthority, services.VerbCreate); err != nil {
+		return trace.Wrap(err)
+	}
+	if err := a.actionWithContext(ctx, defaults.Namespace, services.KindCertAuthority, services.VerbUpdate); err != nil {
+		return trace.Wrap(err)
+	}
 	return a.authServer.UpsertCertAuthority(ca)
 }
 
+func (a *AuthWithRoles) CompareAndSwapCertAuthority(new, existing services.CertAuthority) error {
+	if err := a.action(defaults.Namespace, services.KindCertAuthority, services.VerbCreate); err != nil {
+		return trace.Wrap(err)
+	}
+	if err := a.action(defaults.Namespace, services.KindCertAuthority, services.VerbUpdate); err != nil {
+		return trace.Wrap(err)
+	}
+	return a.authServer.CompareAndSwapCertAuthority(new, existing)
+}
+
 func (a *AuthWithRoles) GetCertAuthorities(caType services.CertAuthType, loadKeys bool) ([]services.CertAuthority, error) {
 	if err := a.action(defaults.Namespace, services.KindCertAuthority, services.VerbList); err != nil {
 		return nil, trace.Wrap(err)
@@ -156,7 +192,6 @@ func (a *AuthWithRoles) GetCertAuthorities(caType services.CertAuthType, loadKey
 			return nil, trace.Wrap(err)
 		}
 	}
-
 	return a.authServer.GetCertAuthorities(caType, loadKeys)
 }
 
@@ -172,13 +207,6 @@ func (a *AuthWithRoles) GetCertAuthority(id services.CertAuthID, loadKeys bool)
 	return a.authServer.GetCertAuthority(id, loadKeys)
 }
 
-func (a *AuthWithRoles) GetAnyCertAuthority(id services.CertAuthID) (services.CertAuthority, error) {
-	if err := a.action(defaults.Namespace, services.KindCertAuthority, services.VerbReadNoSecrets); err != nil {
-		return nil, trace.Wrap(err)
-	}
-	return a.authServer.GetAnyCertAuthority(id)
-}
-
 func (a *AuthWithRoles) GetDomainName() (string, error) {
 	// anyone can read it, no harm in that
 	return a.authServer.GetDomainName()