Skip to content

Commit 95f14a9

Browse files
committed
[FAB-7491] client TLS cert support for gossip
This change set adds client TLS certificate support for gossip. The handshake method now selects the certificate to hash and attach its binding to the handshake message according to whether the peer initiated the connection or not. The change set adds the support in the form of a struct that wraps the TLS certificates with atomic references. This is in order to prepare for the future where we might have dynamic TLS certificate updates. Unit tests haven't been added, because I changed the test code to always have a different client and server certificate, and in case only the server certificate is present than the code that is executed in the production path is code that is already tested, because it is computed by code in core/peer/start.go Change-Id: I1edddb2321c629f88080510befe1db26fa0b6925 Signed-off-by: yacovm <yacovm@il.ibm.com>
1 parent 4d802d1 commit 95f14a9

File tree

12 files changed

+97
-87
lines changed

12 files changed

+97
-87
lines changed

core/peer/peer_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ func TestCreateChainFromBlock(t *testing.T) {
121121
return dialOpts
122122
}
123123
err = service.InitGossipServiceCustomDeliveryFactory(
124-
identity, "localhost:13611", grpcServer,
124+
identity, "localhost:13611", grpcServer, nil,
125125
&mockDeliveryClientFactory{},
126126
messageCryptoService, secAdv, defaultSecureDialOpts)
127127

core/scc/cscc/configure_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ func TestConfigerInvokeJoinChainCorrectParams(t *testing.T) {
220220
identity, _ := mgmt.GetLocalSigningIdentityOrPanic().Serialize()
221221
messageCryptoService := peergossip.NewMCS(&mocks.ChannelPolicyManagerGetter{}, localmsp.NewSigner(), mgmt.NewDeserializersManager())
222222
secAdv := peergossip.NewSecurityAdvisor(mgmt.NewDeserializersManager())
223-
err := service.InitGossipServiceCustomDeliveryFactory(identity, peerEndpoint, nil, &mockDeliveryClientFactory{}, messageCryptoService, secAdv, nil)
223+
err := service.InitGossipServiceCustomDeliveryFactory(identity, peerEndpoint, nil, nil, &mockDeliveryClientFactory{}, messageCryptoService, secAdv, nil)
224224
assert.NoError(t, err)
225225

226226
// Successful path for JoinChain

gossip/comm/comm_impl.go

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,14 @@ func NewCommInstanceWithServer(port int, idMapper identity.Mapper, peerIdentity
5757

5858
var ll net.Listener
5959
var s *grpc.Server
60-
var certHash []byte
60+
var certs *common.TLSCertificates
6161

6262
if port > 0 {
63-
s, ll, secureDialOpts, certHash = createGRPCLayer(port)
63+
s, ll, secureDialOpts, certs = createGRPCLayer(port)
6464
}
6565

6666
commInst := &commImpl{
6767
pubSub: util.NewPubSub(),
68-
selfCertHash: certHash,
6968
PKIID: idMapper.GetPKIidOfCert(peerIdentity),
7069
idMapper: idMapper,
7170
logger: util.GetLogger(util.LoggingCommModule, fmt.Sprintf("%d", port)),
@@ -82,6 +81,7 @@ func NewCommInstanceWithServer(port int, idMapper identity.Mapper, peerIdentity
8281
exitChan: make(chan struct{}, 1),
8382
subscriptions: make([]chan proto.ReceivedMessage, 0),
8483
dialTimeout: util.GetDurationOrDefault("peer.gossip.dialTimeout", defDialTimeout),
84+
tlsCerts: certs,
8585
}
8686
commInst.connStore = newConnStore(commInst, commInst.logger)
8787

@@ -98,7 +98,7 @@ func NewCommInstanceWithServer(port int, idMapper identity.Mapper, peerIdentity
9898
}
9999

100100
// NewCommInstance creates a new comm instance that binds itself to the given gRPC server
101-
func NewCommInstance(s *grpc.Server, cert *tls.Certificate, idStore identity.Mapper,
101+
func NewCommInstance(s *grpc.Server, certs *common.TLSCertificates, idStore identity.Mapper,
102102
peerIdentity api.PeerIdentityType, secureDialOpts api.PeerSecureDialOpts,
103103
dialOpts ...grpc.DialOption) (Comm, error) {
104104

@@ -107,23 +107,16 @@ func NewCommInstance(s *grpc.Server, cert *tls.Certificate, idStore identity.Map
107107
return nil, errors.WithStack(err)
108108
}
109109

110-
if cert != nil {
111-
inst := commInst.(*commImpl)
112-
if len(cert.Certificate) == 0 {
113-
inst.logger.Panic("Certificate supplied but certificate chain is empty")
114-
} else {
115-
inst.selfCertHash = certHashFromRawCert(cert.Certificate[0])
116-
}
117-
}
110+
commInst.(*commImpl).tlsCerts = certs
118111

119112
proto.RegisterGossipServer(s, commInst.(*commImpl))
120113

121114
return commInst, nil
122115
}
123116

124117
type commImpl struct {
118+
tlsCerts *common.TLSCertificates
125119
pubSub *util.PubSub
126-
selfCertHash []byte
127120
peerIdentity api.PeerIdentityType
128121
idMapper identity.Mapper
129122
logger *logging.Logger
@@ -179,7 +172,7 @@ func (c *commImpl) createConnection(endpoint string, expectedPKIID common.PKIidT
179172

180173
ctx, cf := context.WithCancel(context.Background())
181174
if stream, err = cl.GossipStream(ctx); err == nil {
182-
connInfo, err = c.authenticateRemotePeer(stream)
175+
connInfo, err = c.authenticateRemotePeer(stream, true)
183176
if err == nil {
184177
pkiID = connInfo.ID
185178
if expectedPKIID != nil && !bytes.Equal(pkiID, expectedPKIID) {
@@ -301,7 +294,7 @@ func (c *commImpl) Handshake(remotePeer *RemotePeer) (api.PeerIdentityType, erro
301294
if err != nil {
302295
return nil, err
303296
}
304-
connInfo, err := c.authenticateRemotePeer(stream)
297+
connInfo, err := c.authenticateRemotePeer(stream, true)
305298
if err != nil {
306299
c.logger.Warningf("Authentication failed: %v", err)
307300
return nil, err
@@ -402,13 +395,22 @@ func extractRemoteAddress(stream stream) string {
402395
return remoteAddress
403396
}
404397

405-
func (c *commImpl) authenticateRemotePeer(stream stream) (*proto.ConnectionInfo, error) {
398+
func (c *commImpl) authenticateRemotePeer(stream stream, initiator bool) (*proto.ConnectionInfo, error) {
406399
ctx := stream.Context()
407400
remoteAddress := extractRemoteAddress(stream)
408401
remoteCertHash := extractCertificateHashFromContext(ctx)
409402
var err error
410403
var cMsg *proto.SignedGossipMessage
411-
useTLS := c.selfCertHash != nil
404+
useTLS := c.tlsCerts != nil
405+
var selfCertHash []byte
406+
407+
if useTLS {
408+
certReference := c.tlsCerts.TLSServerCert
409+
if initiator {
410+
certReference = c.tlsCerts.TLSClientCert
411+
}
412+
selfCertHash = certHashFromRawCert(certReference.Load().(*tls.Certificate).Certificate[0])
413+
}
412414

413415
signer := func(msg []byte) ([]byte, error) {
414416
return c.idMapper.Sign(msg)
@@ -420,7 +422,7 @@ func (c *commImpl) authenticateRemotePeer(stream stream) (*proto.ConnectionInfo,
420422
return nil, fmt.Errorf("No TLS certificate")
421423
}
422424

423-
cMsg, err = c.createConnectionMsg(c.PKIID, c.selfCertHash, c.peerIdentity, signer)
425+
cMsg, err = c.createConnectionMsg(c.PKIID, selfCertHash, c.peerIdentity, signer)
424426
if err != nil {
425427
return nil, err
426428
}
@@ -545,7 +547,7 @@ func (c *commImpl) GossipStream(stream proto.Gossip_GossipStreamServer) error {
545547
if c.isStopping() {
546548
return fmt.Errorf("Shutting down")
547549
}
548-
connInfo, err := c.authenticateRemotePeer(stream)
550+
connInfo, err := c.authenticateRemotePeer(stream, false)
549551
if err != nil {
550552
c.logger.Errorf("Authentication failed: %v", err)
551553
return err
@@ -653,25 +655,24 @@ type stream interface {
653655
grpc.Stream
654656
}
655657

656-
func createGRPCLayer(port int) (*grpc.Server, net.Listener, api.PeerSecureDialOpts, []byte) {
657-
var returnedCertHash []byte
658+
func createGRPCLayer(port int) (*grpc.Server, net.Listener, api.PeerSecureDialOpts, *common.TLSCertificates) {
658659
var s *grpc.Server
659660
var ll net.Listener
660661
var err error
661662
var serverOpts []grpc.ServerOption
662663
var dialOpts []grpc.DialOption
663664

664-
cert := GenerateCertificatesOrPanic()
665-
returnedCertHash = certHashFromRawCert(cert.Certificate[0])
665+
clientCert := GenerateCertificatesOrPanic()
666+
serverCert := GenerateCertificatesOrPanic()
666667

667668
tlsConf := &tls.Config{
668-
Certificates: []tls.Certificate{cert},
669+
Certificates: []tls.Certificate{serverCert},
669670
ClientAuth: tls.RequestClientCert,
670671
InsecureSkipVerify: true,
671672
}
672673
serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsConf)))
673674
ta := credentials.NewTLS(&tls.Config{
674-
Certificates: []tls.Certificate{cert},
675+
Certificates: []tls.Certificate{clientCert},
675676
InsecureSkipVerify: true,
676677
})
677678
dialOpts = append(dialOpts, grpc.WithTransportCredentials(ta))
@@ -685,7 +686,10 @@ func createGRPCLayer(port int) (*grpc.Server, net.Listener, api.PeerSecureDialOp
685686
return dialOpts
686687
}
687688
s = grpc.NewServer(serverOpts...)
688-
return s, ll, secureDialOpts, returnedCertHash
689+
certs := &common.TLSCertificates{}
690+
certs.TLSServerCert.Store(&serverCert)
691+
certs.TLSClientCert.Store(&clientCert)
692+
return s, ll, secureDialOpts, certs
689693
}
690694

691695
func topicForAck(nonce uint64, pkiID common.PKIidType) string {

gossip/comm/comm_test.go

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -350,22 +350,18 @@ func TestBasic(t *testing.T) {
350350

351351
func TestProdConstructor(t *testing.T) {
352352
t.Parallel()
353-
peerIdentity := GenerateCertificatesOrPanic()
354-
srv, lsnr, dialOpts, certHash := createGRPCLayer(20000)
353+
srv, lsnr, dialOpts, certs := createGRPCLayer(20000)
355354
defer srv.Stop()
356355
defer lsnr.Close()
357356
id := []byte("localhost:20000")
358-
comm1, _ := NewCommInstance(srv, &peerIdentity, identity.NewIdentityMapper(naiveSec, id, noopPurgeIdentity), id, dialOpts)
359-
comm1.(*commImpl).selfCertHash = certHash
357+
comm1, _ := NewCommInstance(srv, certs, identity.NewIdentityMapper(naiveSec, id, noopPurgeIdentity), id, dialOpts)
360358
go srv.Serve(lsnr)
361359

362-
peerIdentity = GenerateCertificatesOrPanic()
363-
srv, lsnr, dialOpts, certHash = createGRPCLayer(30000)
360+
srv, lsnr, dialOpts, certs = createGRPCLayer(30000)
364361
defer srv.Stop()
365362
defer lsnr.Close()
366363
id = []byte("localhost:30000")
367-
comm2, _ := NewCommInstance(srv, &peerIdentity, identity.NewIdentityMapper(naiveSec, id, noopPurgeIdentity), id, dialOpts)
368-
comm2.(*commImpl).selfCertHash = certHash
364+
comm2, _ := NewCommInstance(srv, certs, identity.NewIdentityMapper(naiveSec, id, noopPurgeIdentity), id, dialOpts)
369365
go srv.Serve(lsnr)
370366
defer comm1.Stop()
371367
defer comm2.Stop()

gossip/common/cert.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
Copyright IBM Corp. All Rights Reserved.
3+
4+
SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package common
8+
9+
import (
10+
"sync/atomic"
11+
)
12+
13+
// TLSCertificates aggregates server and client TLS certificates
14+
type TLSCertificates struct {
15+
TLSServerCert atomic.Value // *tls.Certificate server certificate of the peer
16+
TLSClientCert atomic.Value // *tls.Certificate client certificate of the peer
17+
}

gossip/gossip/gossip.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ SPDX-License-Identifier: Apache-2.0
77
package gossip
88

99
import (
10-
"crypto/tls"
1110
"fmt"
1211
"time"
1312

@@ -113,10 +112,11 @@ type Config struct {
113112

114113
SkipBlockVerification bool // Should we skip verifying block messages or not
115114

116-
PublishCertPeriod time.Duration // Time from startup certificates are included in Alive messages
117-
PublishStateInfoInterval time.Duration // Determines frequency of pushing state info messages to peers
118-
RequestStateInfoInterval time.Duration // Determines frequency of pulling state info messages from peers
119-
TLSServerCert *tls.Certificate // TLS certificate of the peer
115+
PublishCertPeriod time.Duration // Time from startup certificates are included in Alive messages
116+
PublishStateInfoInterval time.Duration // Determines frequency of pushing state info messages to peers
117+
RequestStateInfoInterval time.Duration // Determines frequency of pulling state info messages from peers
118+
119+
TLSCerts *common.TLSCertificates // TLS certificates of the peer
120120

121121
InternalEndpoint string // Endpoint we publish to peers in our organization
122122
ExternalEndpoint string // Peer publishes this endpoint instead of SelfEndpoint to foreign organizations

gossip/gossip/gossip_impl.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ package gossip
88

99
import (
1010
"bytes"
11-
"crypto/tls"
1211
"fmt"
1312
"reflect"
1413
"sync"
@@ -97,7 +96,7 @@ func NewGossipService(conf *Config, s *grpc.Server, secAdvisor api.SecurityAdvis
9796
if s == nil {
9897
g.comm, err = createCommWithServer(conf.BindPort, g.idMapper, selfIdentity, secureDialOpts)
9998
} else {
100-
g.comm, err = createCommWithoutServer(s, conf.TLSServerCert, g.idMapper, selfIdentity, secureDialOpts)
99+
g.comm, err = createCommWithoutServer(s, conf.TLSCerts, g.idMapper, selfIdentity, secureDialOpts)
101100
}
102101

103102
if err != nil {
@@ -159,9 +158,9 @@ func newChannelState(g *gossipServiceImpl) *channelState {
159158
}
160159
}
161160

162-
func createCommWithoutServer(s *grpc.Server, cert *tls.Certificate, idStore identity.Mapper,
161+
func createCommWithoutServer(s *grpc.Server, certs *common.TLSCertificates, idStore identity.Mapper,
163162
identity api.PeerIdentityType, secureDialOpts api.PeerSecureDialOpts) (comm.Comm, error) {
164-
return comm.NewCommInstance(s, cert, idStore, identity, secureDialOpts)
163+
return comm.NewCommInstance(s, certs, idStore, identity, secureDialOpts)
165164
}
166165

167166
// NewGossipServiceWithServer creates a new gossip instance with a gRPC server

gossip/integration/integration.go

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,12 @@ SPDX-License-Identifier: Apache-2.0
77
package integration
88

99
import (
10-
"crypto/tls"
1110
"net"
1211
"strconv"
1312
"time"
1413

15-
"github.com/hyperledger/fabric/core/config"
1614
"github.com/hyperledger/fabric/gossip/api"
15+
"github.com/hyperledger/fabric/gossip/common"
1716
"github.com/hyperledger/fabric/gossip/gossip"
1817
"github.com/hyperledger/fabric/gossip/util"
1918
"github.com/pkg/errors"
@@ -23,7 +22,7 @@ import (
2322

2423
// This file is used to bootstrap a gossip instance and/or leader election service instance
2524

26-
func newConfig(selfEndpoint string, externalEndpoint string, bootPeers ...string) (*gossip.Config, error) {
25+
func newConfig(selfEndpoint string, externalEndpoint string, certs *common.TLSCertificates, bootPeers ...string) (*gossip.Config, error) {
2726
_, p, err := net.SplitHostPort(selfEndpoint)
2827

2928
if err != nil {
@@ -35,16 +34,7 @@ func newConfig(selfEndpoint string, externalEndpoint string, bootPeers ...string
3534
return nil, errors.Wrapf(err, "misconfigured endpoint %s, failed to parse port number", selfEndpoint)
3635
}
3736

38-
var cert *tls.Certificate
39-
if viper.GetBool("peer.tls.enabled") {
40-
certTmp, err := tls.LoadX509KeyPair(config.GetPath("peer.tls.cert.file"), config.GetPath("peer.tls.key.file"))
41-
if err != nil {
42-
return nil, errors.Wrap(err, "failed to load certificates")
43-
}
44-
cert = &certTmp
45-
}
46-
47-
return &gossip.Config{
37+
conf := &gossip.Config{
4838
BindPort: int(port),
4939
BootstrapPeers: bootPeers,
5040
ID: selfEndpoint,
@@ -61,18 +51,20 @@ func newConfig(selfEndpoint string, externalEndpoint string, bootPeers ...string
6151
RequestStateInfoInterval: util.GetDurationOrDefault("peer.gossip.requestStateInfoInterval", 4*time.Second),
6252
PublishStateInfoInterval: util.GetDurationOrDefault("peer.gossip.publishStateInfoInterval", 4*time.Second),
6353
SkipBlockVerification: viper.GetBool("peer.gossip.skipBlockVerification"),
64-
TLSServerCert: cert,
65-
}, nil
54+
TLSCerts: certs,
55+
}
56+
57+
return conf, nil
6658
}
6759

6860
// NewGossipComponent creates a gossip component that attaches itself to the given gRPC server
6961
func NewGossipComponent(peerIdentity []byte, endpoint string, s *grpc.Server,
7062
secAdv api.SecurityAdvisor, cryptSvc api.MessageCryptoService,
71-
secureDialOpts api.PeerSecureDialOpts, bootPeers ...string) (gossip.Gossip, error) {
63+
secureDialOpts api.PeerSecureDialOpts, certs *common.TLSCertificates, bootPeers ...string) (gossip.Gossip, error) {
7264

7365
externalEndpoint := viper.GetString("peer.gossip.externalEndpoint")
7466

75-
conf, err := newConfig(endpoint, externalEndpoint, bootPeers...)
67+
conf, err := newConfig(endpoint, externalEndpoint, certs, bootPeers...)
7668
if err != nil {
7769
return nil, errors.WithStack(err)
7870
}

gossip/integration/integration_test.go

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@ func TestNewGossipCryptoService(t *testing.T) {
5454
msptesttools.LoadMSPSetupForTesting()
5555
peerIdentity, _ := mgmt.GetLocalSigningIdentityOrPanic().Serialize()
5656
g1, err := NewGossipComponent(peerIdentity, endpoint1, s1, secAdv, cryptSvc,
57-
defaultSecureDialOpts)
57+
defaultSecureDialOpts, nil)
5858
assert.NoError(t, err)
5959
g2, err := NewGossipComponent(peerIdentity, endpoint2, s2, secAdv, cryptSvc,
60-
defaultSecureDialOpts, endpoint1)
60+
defaultSecureDialOpts, nil, endpoint1)
6161
assert.NoError(t, err)
6262
g3, err := NewGossipComponent(peerIdentity, endpoint3, s3, secAdv, cryptSvc,
63-
defaultSecureDialOpts, endpoint1)
63+
defaultSecureDialOpts, nil, endpoint1)
6464
assert.NoError(t, err)
6565
defer g1.Stop()
6666
defer g2.Stop()
@@ -70,18 +70,6 @@ func TestNewGossipCryptoService(t *testing.T) {
7070
go s3.Serve(ll3)
7171
}
7272

73-
func TestBadInitialization(t *testing.T) {
74-
msptesttools.LoadMSPSetupForTesting()
75-
peerIdentity, _ := mgmt.GetLocalSigningIdentityOrPanic().Serialize()
76-
s1 := grpc.NewServer()
77-
_, err := newConfig("anEndpointWithoutAPort", "anEndpointWithoutAPort")
78-
79-
viper.Set("peer.tls.enabled", true)
80-
_, err = NewGossipComponent(peerIdentity, "localhost:5000", s1, secAdv, cryptSvc,
81-
defaultSecureDialOpts)
82-
assert.Error(t, err)
83-
}
84-
8573
func setupTestEnv() {
8674
viper.SetConfigName("core")
8775
viper.SetEnvPrefix("CORE")

0 commit comments

Comments
 (0)