CS: Store customer keys in trust database (#2241)

Only use files to create the initial state and from then on work with the DB.

go/Makefile

@@ -37,7 +37,8 @@ MOCK_SRC_IN =\
     lib/snet/snetproxy/reconnecter.go \
     lib/snet/snetproxy/io.go \
     lib/log/wrappers.go \
-    lib/sciond/sciond.go
+    lib/sciond/sciond.go \
+    lib/infra/modules/trust/trustdb/trustdb.go
 MOCK_SRC_OUT = $(foreach in,$(MOCK_SRC_IN),$(call mock_src_in_to_out,$(in)))
 # This uses the format <stdlib package>/<interface>:
 MOCK_STD_IN = net/Addr net/PacketConn

go/cert_srv/internal/csconfig/customers.go

@@ -15,47 +15,32 @@
 package csconfig
 
 import (
-	"bytes"
-	"encoding/base64"
+	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
-	"sync"
+	"time"
 
 	"github.com/scionproto/scion/go/lib/addr"
 	"github.com/scionproto/scion/go/lib/common"
+	"github.com/scionproto/scion/go/lib/infra/modules/trust/trustdb"
 	"github.com/scionproto/scion/go/lib/keyconf"
 )
 
 const (
-	KeyChanged   = "Verifying key has changed in the meantime"
-	NotACustomer = "ISD-AS not in customer mapping"
-
 	CustomersDir = "customers"
 )
 
 // reCustVerKey is used to parse the IA and version of a customer verifying key file.
 var reCustVerKey = regexp.MustCompile(`^(ISD\S+-AS\S+)-V(\d+)\.key$`)
 
-// Customers is a mapping from non-core ASes assigned to this core AS to their public
-// verifying key.
-type Customers struct {
-	m sync.RWMutex
-	// custMap is the customer mapping.
-	custMap map[addr.IA]common.RawBytes
-	// path is the path to the customers directory.
-	path string
-}
-
-// loadCustomers populates the mapping from assigned non-core ASes to their verifying key.
-func (s *State) loadCustomers(stateDir string) (*Customers, error) {
-	cust := &Customers{path: filepath.Join(stateDir, CustomersDir)}
-	files, err := filepath.Glob(fmt.Sprintf("%s/ISD*-AS*-V*.key", cust.path))
+// LoadCustomers populates the DB from assigned non-core ASes to their verifying key.
+func LoadCustomers(stateDir string, trustDB trustdb.TrustDB) error {
+	path := filepath.Join(stateDir, CustomersDir)
+	files, err := filepath.Glob(fmt.Sprintf("%s/ISD*-AS*-V*.key", path))
 	if err != nil {
-		return nil, err
+		return err
 	}
 	activeKeys := make(map[addr.IA]string)
 	activeVers := make(map[addr.IA]uint64)
@@ -64,68 +49,36 @@ func (s *State) loadCustomers(stateDir string) (*Customers, error) {
 		s := reCustVerKey.FindStringSubmatch(name)
 		ia, err := addr.IAFromFileFmt(s[1], true)
 		if err != nil {
-			return nil, common.NewBasicError("Unable to parse IA", err, "file", file)
+			return common.NewBasicError("Unable to parse IA", err, "file", file)
 		}
 		ver, err := strconv.ParseUint(s[2], 10, 64)
 		if err != nil {
-			return nil, common.NewBasicError("Unable to parse Version", err, "file", file)
+			return common.NewBasicError("Unable to parse Version", err, "file", file)
 		}
 		if ver >= activeVers[ia] {
 			activeKeys[ia] = file
 			activeVers[ia] = ver
 		}
 	}
-	cust.custMap = make(map[addr.IA]common.RawBytes)
+	ctx, cancelF := context.WithTimeout(context.Background(), time.Second)
+	defer cancelF()
 	for ia, file := range activeKeys {
 		key, err := keyconf.LoadKey(file, keyconf.RawKey)
 		if err != nil {
-			return nil, common.NewBasicError("Unable to load key", err, "file", file)
+			return common.NewBasicError("Unable to load key", err, "file", file)
 		}
-		cust.custMap[ia] = key
-	}
-	return cust, nil
-}
-
-// GetVerifyingKey returns the verifying key from the requested AS and nil if it is in the mapping.
-// Otherwise, nil and an error.
-func (c *Customers) GetVerifyingKey(ia addr.IA) (common.RawBytes, error) {
-	c.m.RLock()
-	defer c.m.RUnlock()
-	b, ok := c.custMap[ia]
-	if !ok {
-		return nil, common.NewBasicError(NotACustomer, nil, "ISD-AS", ia)
-	}
-	return b, nil
-}
-
-// SetVerifyingKey sets the verifying key for a specified AS. The key is written to the file system.
-func (c *Customers) SetVerifyingKey(ia addr.IA, ver uint64, newKey, oldKey common.RawBytes) error {
-	c.m.Lock()
-	defer c.m.Unlock()
-	currKey, ok := c.custMap[ia]
-	if !ok {
-		return common.NewBasicError(NotACustomer, nil, "ISD-AS", ia)
-	}
-	// Check that the key in the mapping has not changed in the mean time
-	if !bytes.Equal(currKey, oldKey) {
-		return common.NewBasicError(KeyChanged, nil, "ISD-AS", ia)
-	}
-	// Key has to be written to file system, only if it has changed
-	if !bytes.Equal(newKey, currKey) {
-		var err error
-		name := fmt.Sprintf("%s-V%d.key", ia.FileFmt(true), ver)
-		path := filepath.Join(c.path, name)
-		if _, err = os.Stat(path); !os.IsNotExist(err) {
-			return err
+		_, dbV, err := trustDB.GetCustKey(ctx, ia)
+		if err != nil {
+			return common.NewBasicError("Failed to check DB cust key", err, "ia", ia)
+		}
+		if dbV >= activeVers[ia] {
+			// db already contains a newer key.
+			continue
 		}
-		buf := make([]byte, base64.StdEncoding.EncodedLen(len(newKey)))
-		base64.StdEncoding.Encode(buf, newKey)
-		if err = ioutil.WriteFile(path, buf, 0644); err != nil {
-			return err
+		err = trustDB.InsertCustKey(ctx, ia, activeVers[ia], key, dbV)
+		if err != nil {
+			return common.NewBasicError("Failed to save customer key", err, "file", file)
 		}
-		c.custMap[ia] = make(common.RawBytes, len(newKey))
-		copy(c.custMap[ia], newKey)
-		return nil
 	}
 	return nil
 }

go/cert_srv/internal/csconfig/state.go

@@ -34,9 +34,6 @@ type State struct {
 	keyConf *keyconf.Conf
 	// keyConfLock guards KeyConf.
 	keyConfLock sync.RWMutex
-	// Customers is a mapping from non-core ASes assigned to this core AS to their public
-	// verifying key.
-	Customers *Customers
 	// signer is used to sign ctrl payloads.
 	signer ctrl.Signer
 	// signerLock guards signer.
@@ -47,14 +44,18 @@ type State struct {
 	verifierLock sync.RWMutex
 }
 
-func LoadState(confDir string, isCore bool) (*State, error) {
-	s := &State{}
+func LoadState(confDir string, isCore bool, trustDB trustdb.TrustDB,
+	trustStore *trust.Store) (*State, error) {
+
+	s := &State{
+		Store:   trustStore,
+		TrustDB: trustDB,
+	}
 	if err := s.loadKeyConf(confDir, isCore); err != nil {
 		return nil, err
 	}
 	if isCore {
-		var err error
-		if s.Customers, err = s.loadCustomers(confDir); err != nil {
+		if err := LoadCustomers(confDir, s.TrustDB); err != nil {
 			return nil, common.NewBasicError(ErrorCustomers, err)
 		}
 	}

go/cert_srv/internal/csconfig/state_test.go

@@ -17,10 +17,14 @@ package csconfig
 import (
 	"testing"
 
+	"github.com/golang/mock/gomock"
 	. "github.com/smartystreets/goconvey/convey"
 
+	"github.com/scionproto/scion/go/lib/common"
+	"github.com/scionproto/scion/go/lib/infra/modules/trust/trustdb/mock_trustdb"
 	"github.com/scionproto/scion/go/lib/keyconf"
 	"github.com/scionproto/scion/go/lib/scrypto"
+	"github.com/scionproto/scion/go/lib/xtest"
 )
 
 var (
@@ -33,8 +37,16 @@ var (
 )
 
 func TestLoadState(t *testing.T) {
+	key := common.RawBytes([]byte("aaaaaaa"))
 	Convey("Load core state", t, func() {
-		state, err := LoadState("testdata", true)
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+		trustDB := mock_trustdb.NewMockTrustDB(ctrl)
+		ia := xtest.MustParseIA("1-ff00:0:110")
+		trustDB.EXPECT().GetCustKey(gomock.Any(), gomock.Eq(ia)).Return(nil, uint64(0), nil)
+		trustDB.EXPECT().InsertCustKey(gomock.Any(), gomock.Eq(ia), uint64(1),
+			gomock.Eq(key), uint64(0))
+		state, err := LoadState("testdata", true, trustDB, nil)
 		SoMsg("err", err, ShouldBeNil)
 		SoMsg("Master0", state.keyConf.Master.Key0, ShouldResemble, mstr0)
 		SoMsg("Master1", state.keyConf.Master.Key1, ShouldResemble, mstr1)
@@ -46,7 +58,7 @@ func TestLoadState(t *testing.T) {
 	})
 
 	Convey("Load non-core state", t, func() {
-		state, err := LoadState("testdata", false)
+		state, err := LoadState("testdata", false, nil, nil)
 		SoMsg("err", err, ShouldBeNil)
 		SoMsg("Master0", state.keyConf.Master.Key0, ShouldResemble, mstr0)
 		SoMsg("Master1", state.keyConf.Master.Key1, ShouldResemble, mstr1)

go/cert_srv/internal/csconfig/testdata/customers/ISD1-ASff00_0_110-V1.key

@@ -0,0 +1 @@
+YWFhYWFhYQ==

go/cert_srv/internal/reiss/handler.go

@@ -35,6 +35,9 @@ import (
 
 const (
 	HandlerTimeout = 5 * time.Second
+
+	KeyChanged   = "Verifying key has changed in the meantime"
+	NotACustomer = "ISD-AS not in customer mapping"
 )
 
 // Handler handles certificate chain reissue requests.
@@ -85,7 +88,7 @@ func (h *Handler) handle(r *infra.Request, addr *snet.Addr, req *cert_mgmt.Chain
 		return h.sendRep(ctx, addr, maxChain, r.ID)
 	}
 	// Get the verifying key from the customer mapping
-	verKey, err := h.State.Customers.GetVerifyingKey(addr.IA)
+	verKey, verVersion, err := h.getVerifyingKey(ctx, addr.IA)
 	if err != nil {
 		return common.NewBasicError("Unable to get verifying key", err)
 	}
@@ -94,7 +97,7 @@ func (h *Handler) handle(r *infra.Request, addr *snet.Addr, req *cert_mgmt.Chain
 		return common.NewBasicError("Unable to verify request", err)
 	}
 	// Issue certificate chain
-	newChain, err := h.issueChain(ctx, crt, verKey)
+	newChain, err := h.issueChain(ctx, crt, verKey, verVersion)
 	if err != nil {
 		return common.NewBasicError("Unable to reissue certificate chain", err)
 	}
@@ -143,7 +146,7 @@ func (h *Handler) validateReq(c *cert.Certificate, vKey common.RawBytes,
 			vChain.Leaf.Subject, "sub", c.Subject)
 	}
 	if maxChain.Leaf.Version+1 != c.Version {
-		return common.NewBasicError("Invalid version", nil, "expected", maxChain.Leaf.Version,
+		return common.NewBasicError("Invalid version", nil, "expected", maxChain.Leaf.Version+1,
 			"actual", c.Version)
 	}
 	if !c.Issuer.Eq(h.IA) {
@@ -162,7 +165,7 @@ func (h *Handler) validateReq(c *cert.Certificate, vKey common.RawBytes,
 // issueChain creates a certificate chain for the certificate and adds it to the
 // trust store.
 func (h *Handler) issueChain(ctx context.Context, c *cert.Certificate,
-	vKey common.RawBytes) (*cert.Chain, error) {
+	vKey common.RawBytes, verVersion uint64) (*cert.Chain, error) {
 
 	issCert, err := h.getIssuerCert(ctx)
 	if err != nil {
@@ -184,15 +187,29 @@ func (h *Handler) issueChain(ctx context.Context, c *cert.Certificate,
 	if err != nil {
 		return nil, err
 	}
+	tx, err := h.State.TrustDB.BeginTransaction(ctx, nil)
+	if err != nil {
+		return nil, common.NewBasicError("Failed to create transaction", err)
+	}
 	// Set verifying key.
-	err = h.State.Customers.SetVerifyingKey(c.Subject, c.Version, c.SubjectSignKey, vKey)
+	err = tx.InsertCustKey(ctx, c.Subject, c.Version, c.SubjectSignKey, verVersion)
 	if err != nil {
+		tx.Rollback()
 		return nil, err
 	}
-	if _, err = h.State.TrustDB.InsertChain(ctx, chain); err != nil {
+	var n int64
+	if n, err = tx.InsertChain(ctx, chain); err != nil {
+		tx.Rollback()
 		log.Error("[ReissHandler] Unable to write reissued certificate chain to disk", "err", err)
 		return nil, err
 	}
+	if n == 0 {
+		tx.Rollback()
+		return nil, common.NewBasicError("Chain already in DB", nil, "chain", chain)
+	}
+	if err = tx.Commit(); err != nil {
+		return nil, common.NewBasicError("Failed to commit transaction", err)
+	}
 	return chain, nil
 }
 
@@ -222,3 +239,18 @@ func (h *Handler) getIssuerCert(ctx context.Context) (*cert.Certificate, error)
 	}
 	return issCrt, nil
 }
+
+// getVerifyingKey returns the verifying key from the requested AS and nil if it is in the mapping.
+// Otherwise, nil and an error.
+func (h *Handler) getVerifyingKey(ctx context.Context,
+	ia addr.IA) (common.RawBytes, uint64, error) {
+
+	k, v, err := h.State.TrustDB.GetCustKey(ctx, ia)
+	if err != nil {
+		return nil, 0, err
+	}
+	if k == nil {
+		return nil, 0, common.NewBasicError(NotACustomer, nil, "ISD-AS", ia)
+	}
+	return k, v, nil
+}

go/cert_srv/setup.go

@@ -77,23 +77,24 @@ func setup() error {
 
 // initState sets the state.
 func initState(config *Config) error {
-	var err error
-	config.state, err = csconfig.LoadState(config.General.ConfigDir, config.General.Topology.Core)
+	trustDB, err := config.TrustDB.New()
 	if err != nil {
-		return common.NewBasicError("Unable to load CS state", err)
-	}
-	if config.state.TrustDB, err = config.TrustDB.New(); err != nil {
 		return common.NewBasicError("Unable to initialize trustDB", err)
 	}
 	trustConf := &trust.Config{
 		MustHaveLocalChain: true,
 		ServiceType:        proto.ServiceType_cs,
 	}
-	config.state.Store, err = trust.NewStore(config.state.TrustDB,
-		config.General.Topology.ISD_AS, trustConf, log.Root())
+	trustStore, err := trust.NewStore(trustDB, config.General.Topology.ISD_AS,
+		trustConf, log.Root())
 	if err != nil {
 		return common.NewBasicError("Unable to initialize trust store", err)
 	}
+	config.state, err = csconfig.LoadState(config.General.ConfigDir, config.General.Topology.Core,
+		trustDB, trustStore)
+	if err != nil {
+		return common.NewBasicError("Unable to load CS state", err)
+	}
 	err = config.state.Store.LoadAuthoritativeTRC(filepath.Join(config.General.ConfigDir, "certs"))
 	if err != nil {
 		return common.NewBasicError("Unable to load local TRC", err)

go/lib/infra/modules/trust/trustdb/trustdb.go

@@ -22,6 +22,7 @@ import (
 	"io"
 
 	"github.com/scionproto/scion/go/lib/addr"
+	"github.com/scionproto/scion/go/lib/common"
 	"github.com/scionproto/scion/go/lib/scrypto/cert"
 	"github.com/scionproto/scion/go/lib/scrypto/trc"
 )
@@ -65,6 +66,8 @@ type Read interface {
 	GetTRCMaxVersion(ctx context.Context, isd addr.ISD) (*trc.TRC, error)
 	// GetAllTRCs fetches all TRCs from the database.
 	GetAllTRCs(ctx context.Context) ([]*trc.TRC, error)
+	// GetCustKey gets the latest signing key and version for the specified customer AS.
+	GetCustKey(ctx context.Context, ia addr.IA) (common.RawBytes, uint64, error)
 }
 
 // Write contains all write operations fo the trust DB.
@@ -79,6 +82,13 @@ type Write interface {
 	// InsertTRC inserts trcobj into the database. The first return value is the
 	// number of rows affected.
 	InsertTRC(ctx context.Context, trcobj *trc.TRC) (int64, error)
+	// InsertCustKey inserts or updates the given customer key.
+	// If there has been a concurrent insert, i.e. the version in the DB is no longer oldVersion
+	// this operation should return an error.
+	// If there is no previous version 0 should be passed for the oldVersion argument.
+	// If oldVersion == version an error is returned.
+	InsertCustKey(ctx context.Context, ia addr.IA, version uint64,
+		key common.RawBytes, oldVersion uint64) error
 }
 
 // Transaction represents a trust DB transaction with an ongoing transaction.