From 63e1563f6274bc2dc6d0de7f54f677eb473fd83b Mon Sep 17 00:00:00 2001 From: Jason Yellick Date: Tue, 5 Nov 2019 13:10:16 -0500 Subject: [PATCH 1/2] FAB-16544 Fix IT UpdateChannel to match doc The documentation in UpdateChannel function says that it waits for the config to commit at all of the peers, but this isn't actually the case. Instead, it waits for the config to commit at the orderer it's already committed at. This change already existed in master, but was apparently not backported to release-1.4. Signed-off-by: Jason Yellick Change-Id: I83a9f78bba37181b45dd5b3ed4e9af23f3556661 --- integration/discovery/discovery_test.go | 2 +- integration/e2e/acl_test.go | 2 +- integration/nwo/configblock.go | 17 ++++++++++++++--- integration/nwo/deploy.go | 2 +- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/integration/discovery/discovery_test.go b/integration/discovery/discovery_test.go index 7e3bf8624b2..cde78bddf33 100644 --- a/integration/discovery/discovery_test.go +++ b/integration/discovery/discovery_test.go @@ -180,7 +180,7 @@ var _ = Describe("DiscoveryService", func() { currentConfig := nwo.GetConfig(network, network.Peer("org3", "peer0"), orderer, "testchannel") updatedConfig := proto.Clone(currentConfig).(*common.Config) updatedConfig.ChannelGroup.Groups["Application"].Groups["org3"].Policies["Writers"].Policy.Value = utils.MarshalOrPanic(cauthdsl.SignedByMspAdmin("Org3MSP")) - nwo.UpdateConfig(network, orderer, "testchannel", currentConfig, updatedConfig, network.Peer("org3", "peer0")) + nwo.UpdateConfig(network, orderer, "testchannel", currentConfig, updatedConfig, true, network.Peer("org3", "peer0")) By("trying to discover peers as an org 3 member") endorsers = commands.Endorsers{ diff --git a/integration/e2e/acl_test.go b/integration/e2e/acl_test.go index a00fd3acb33..1fa96f16c23 100644 --- a/integration/e2e/acl_test.go +++ b/integration/e2e/acl_test.go @@ -261,7 +261,7 @@ func SetACLPolicy(network *nwo.Network, channel, policyName, policy string, orde }), } - nwo.UpdateConfig(network, orderer, channel, config, updatedConfig, submitter, signer) + nwo.UpdateConfig(network, orderer, channel, config, updatedConfig, true, submitter, signer) } // GetTxIDFromBlock gets a transaction id from a block that has been diff --git a/integration/nwo/configblock.go b/integration/nwo/configblock.go index 15a8ffa086d..1c9d069a21f 100644 --- a/integration/nwo/configblock.go +++ b/integration/nwo/configblock.go @@ -72,7 +72,7 @@ func GetConfig(n *Network, peer *Peer, orderer *Orderer, channel string) *common // UpdateConfig computes, signs, and submits a configuration update and waits // for the update to complete. -func UpdateConfig(n *Network, orderer *Orderer, channel string, current, updated *common.Config, submitter *Peer, additionalSigners ...*Peer) { +func UpdateConfig(n *Network, orderer *Orderer, channel string, current, updated *common.Config, getConfigBlockFromOrderer bool, submitter *Peer, additionalSigners ...*Peer) { tempDir, err := ioutil.TempDir("", "updateConfig") Expect(err).NotTo(HaveOccurred()) defer os.RemoveAll(tempDir) @@ -103,8 +103,13 @@ func UpdateConfig(n *Network, orderer *Orderer, channel string, current, updated Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) } + var currentBlockNumber uint64 // get current configuration block number - currentBlockNumber := CurrentConfigBlockNumber(n, submitter, orderer, channel) + if getConfigBlockFromOrderer { + currentBlockNumber = CurrentConfigBlockNumber(n, submitter, orderer, channel) + } else { + currentBlockNumber = CurrentConfigBlockNumber(n, submitter, nil, channel) + } sess, err := n.PeerAdminSession(submitter, commands.ChannelUpdate{ ChannelID: channel, @@ -115,10 +120,16 @@ func UpdateConfig(n *Network, orderer *Orderer, channel string, current, updated Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) Expect(sess.Err).To(gbytes.Say("Successfully submitted channel update")) + if getConfigBlockFromOrderer { + ccb := func() uint64 { return CurrentConfigBlockNumber(n, submitter, orderer, channel) } + Eventually(ccb, n.EventuallyTimeout).Should(BeNumerically(">", currentBlockNumber)) + return + } + // wait for the block to be committed to all peers that // have joined the channel for _, peer := range n.PeersWithChannel(channel) { - ccb := func() uint64 { return CurrentConfigBlockNumber(n, peer, orderer, channel) } + ccb := func() uint64 { return CurrentConfigBlockNumber(n, peer, nil, channel) } Eventually(ccb, n.EventuallyTimeout).Should(BeNumerically(">", currentBlockNumber)) } } diff --git a/integration/nwo/deploy.go b/integration/nwo/deploy.go index cc808a87ea9..dc45d6d2ffa 100644 --- a/integration/nwo/deploy.go +++ b/integration/nwo/deploy.go @@ -182,7 +182,7 @@ func EnableCapabilities(network *Network, channel, capabilitiesGroup, capabiliti ), } - UpdateConfig(network, orderer, channel, config, updatedConfig, peers[0], peers...) + UpdateConfig(network, orderer, channel, config, updatedConfig, false, peers[0], peers...) } // EnableCapabilitiesOrdererAdmin enables a specific capabilities flag for a running network, From 1a1c71a9004b9f44c15fd2a918849f57abe2fe34 Mon Sep 17 00:00:00 2001 From: Jason Yellick Date: Tue, 5 Nov 2019 13:46:00 -0500 Subject: [PATCH 2/2] FAB-16544 Backport orderer TLS cert overrides This is a partial backport of the orderer endpoint TLS CA overrides present in v2.0. Presently, in v1.4.x because of the code structure, it's very difficult to associate a particular TLS CA with a connection to a particular orderer. So, instead v1.4.x took the approach of adding global TLS overrides. However, moving forward into the v2.0 branch, things have been restructured significantly and it's now possible to bind orderer addresses and TLS CAs. This CR takes the approach of allowing and honoring the TLS CA overrides, but treating them in the same pooled fashion as the previous approach. So, althought he config binds the TLS CAs to a particular address (which will be honored in 2.0), they are all thrown into a global pool for v1.4.x. Signed-off-by: Jason Yellick Change-Id: Ide0d1e75389e1373cc1d92924f01f87684aa6990 --- core/comm/connection.go | 10 ++++++ core/comm/connection_test.go | 10 +++--- core/deliverservice/deliveryclient.go | 14 ++++---- core/deliverservice/deliveryclient_test.go | 22 +++++++----- core/peer/config.go | 26 ++++++++++---- core/peer/config_test.go | 42 +++++++++++++++++++--- gossip/service/gossip_service.go | 3 +- 7 files changed, 96 insertions(+), 31 deletions(-) diff --git a/core/comm/connection.go b/core/comm/connection.go index fa54fbf97db..821d367b75b 100644 --- a/core/comm/connection.go +++ b/core/comm/connection.go @@ -41,6 +41,11 @@ type PerOrgCertificateBundle map[string]CertificateBundle // channel --> organization --> certificates type OrgRootCAs map[string]PerOrgCertificateBundle +type OrdererEndpoint struct { + Address string + PEMs []byte +} + // CertificatesByChannelAndOrg returns the certificates of the given organization in the context // of the given channel. func (orc OrgRootCAs) CertificatesByChannelAndOrg(channel string, org string) CertificateBundle { @@ -104,6 +109,7 @@ func (cs *CredentialSupport) GetDeliverServiceCredentials( channelID string, appendStaticRoots bool, orgs []string, + endpointOverrides map[string]*OrdererEndpoint, ) (credentials.TransportCredentials, error) { cs.RLock() defer cs.RUnlock() @@ -145,6 +151,10 @@ func (cs *CredentialSupport) GetDeliverServiceCredentials( } } + for _, override := range endpointOverrides { + certPool.AppendCertsFromPEM(override.PEMs) + } + // Finally, create a TLS client config with the computed TLS root CAs. var creds credentials.TransportCredentials tlsConfig := &tls.Config{ diff --git a/core/comm/connection_test.go b/core/comm/connection_test.go index ef94747d19c..1c826110c49 100644 --- a/core/comm/connection_test.go +++ b/core/comm/connection_test.go @@ -178,20 +178,20 @@ func TestCredentialSupport(t *testing.T) { cs.ServerRootCAs = [][]byte{rootCAs[5]} cs.ClientRootCAs = [][]byte{rootCAs[5]} - creds, _ := cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"}) + creds, _ := cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"}, nil) assert.Equal(t, "1.2", creds.Info().SecurityVersion, "Expected Security version to be 1.2") creds = cs.GetPeerCredentials() assert.Equal(t, "1.2", creds.Info().SecurityVersion, "Expected Security version to be 1.2") - _, err := cs.GetDeliverServiceCredentials("channel99", false, nil) + _, err := cs.GetDeliverServiceCredentials("channel99", false, nil, nil) assert.EqualError(t, err, "didn't find any root CA certs for channel channel99") // append some bad certs and make sure things still work cs.ServerRootCAs = append(cs.ServerRootCAs, []byte("badcert")) cs.ServerRootCAs = append(cs.ServerRootCAs, []byte(badPEM)) - creds, _ = cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"}) + creds, _ = cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"}, nil) assert.Equal(t, "1.2", creds.Info().SecurityVersion, "Expected Security version to be 1.2") creds = cs.GetPeerCredentials() @@ -284,7 +284,7 @@ func TestImpersonation(t *testing.T) { AppRootCAsByChain: make(map[string]CertificateBundle), OrdererRootCAsByChainAndOrg: make(OrgRootCAs), } - _, err := cs.GetDeliverServiceCredentials("C", false, []string{"SampleOrg"}) + _, err := cs.GetDeliverServiceCredentials("C", false, []string{"SampleOrg"}, nil) assert.Error(t, err) cs.OrdererRootCAsByChainAndOrg.AppendCertificates("A", "SampleOrg", [][]byte{osA.caCert}) @@ -307,7 +307,7 @@ func testInvoke( staticRoots bool, shouldSucceed bool) { - creds, err := cs.GetDeliverServiceCredentials(channelID, staticRoots, []string{"SampleOrg"}) + creds, err := cs.GetDeliverServiceCredentials(channelID, staticRoots, []string{"SampleOrg"}, nil) assert.NoError(t, err) endpoint := s.address diff --git a/core/deliverservice/deliveryclient.go b/core/deliverservice/deliveryclient.go index 05b204ecf6b..1dd5b6ed067 100644 --- a/core/deliverservice/deliveryclient.go +++ b/core/deliverservice/deliveryclient.go @@ -89,7 +89,7 @@ type deliverClient struct { // and how it disseminates the messages to other peers type Config struct { // ConnFactory returns a function that creates a connection to an endpoint - ConnFactory func(channelID string) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error) + ConnFactory func(channelID string, endpointOverrides map[string]*comm.OrdererEndpoint) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error) // ABCFactory creates an AtomicBroadcastClient out of a connection ABCFactory func(*grpc.ClientConn) orderer.AtomicBroadcastClient // CryptoSvc performs cryptographic actions like message verification and signing @@ -111,7 +111,7 @@ type ConnectionCriteria struct { // OrdererEndpointOverrides specifies a map of endpoints to override. The map // key is the configured endpoint address to match and the value is the // endpoint to use instead. - OrdererEndpointOverrides map[string]string + OrdererEndpointOverrides map[string]*comm.OrdererEndpoint } func (cc ConnectionCriteria) toEndpointCriteria() []comm.EndpointCriteria { @@ -128,7 +128,7 @@ func (cc ConnectionCriteria) toEndpointCriteria() []comm.EndpointCriteria { for _, endpoint := range endpoints { // check if we need to override the endpoint if override, ok := cc.OrdererEndpointOverrides[endpoint]; ok { - endpoint = override + endpoint = override.Address } res = append(res, comm.EndpointCriteria{ Organizations: []string{org}, @@ -145,7 +145,7 @@ func (cc ConnectionCriteria) toEndpointCriteria() []comm.EndpointCriteria { for _, endpoint := range cc.OrdererEndpoints { // check if we need to override the endpoint if override, ok := cc.OrdererEndpointOverrides[endpoint]; ok { - endpoint = override + endpoint = override.Address } res = append(res, comm.EndpointCriteria{ Organizations: cc.Organizations, @@ -298,13 +298,13 @@ func (d *deliverServiceImpl) newClient(chainID string, ledgerInfoProvider blocks attempt := float64(attemptNum) return time.Duration(math.Min(math.Pow(2, attempt)*sleepIncrement, reconnectBackoffThreshold)), true } - connProd := comm.NewConnectionProducer(d.conf.ConnFactory(chainID), d.connConfig.toEndpointCriteria()) + connProd := comm.NewConnectionProducer(d.conf.ConnFactory(chainID, d.connConfig.OrdererEndpointOverrides), d.connConfig.toEndpointCriteria()) bClient := NewBroadcastClient(connProd, d.conf.ABCFactory, broadcastSetup, backoffPolicy) requester.client = bClient return bClient } -func DefaultConnectionFactory(channelID string) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error) { +func DefaultConnectionFactory(channelID string, endpointOverrides map[string]*comm.OrdererEndpoint) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error) { return func(criteria comm.EndpointCriteria) (*grpc.ClientConn, error) { dialOpts := []grpc.DialOption{grpc.WithBlock()} // set max send/recv msg sizes @@ -323,7 +323,7 @@ func DefaultConnectionFactory(channelID string) func(endpointCriteria comm.Endpo dialOpts = append(dialOpts, comm.ClientKeepaliveOptions(kaOpts)...) if viper.GetBool("peer.tls.enabled") { - creds, err := comm.GetCredentialSupport().GetDeliverServiceCredentials(channelID, staticRootsEnabled(), criteria.Organizations) + creds, err := comm.GetCredentialSupport().GetDeliverServiceCredentials(channelID, staticRootsEnabled(), criteria.Organizations, endpointOverrides) if err != nil { return nil, fmt.Errorf("failed obtaining credentials for channel %s: %v", channelID, err) } diff --git a/core/deliverservice/deliveryclient_test.go b/core/deliverservice/deliveryclient_test.go index e8c2bbc6834..078b8954820 100644 --- a/core/deliverservice/deliveryclient_test.go +++ b/core/deliverservice/deliveryclient_test.go @@ -102,7 +102,7 @@ func TestNewDeliverService(t *testing.T) { } } - connFactory := func(_ string) func(comm.EndpointCriteria) (*grpc.ClientConn, error) { + connFactory := func(string, map[string]*comm.OrdererEndpoint) func(comm.EndpointCriteria) (*grpc.ClientConn, error) { return func(endpoint comm.EndpointCriteria) (*grpc.ClientConn, error) { lock.Lock() defer lock.Unlock() @@ -277,8 +277,10 @@ func TestDeliverServiceUpdateEndpoints(t *testing.T) { OrdererEndpointsByOrg: map[string][]string{ "org1": {"localhost:5612"}, }, - OrdererEndpointOverrides: map[string]string{ - "localhost:5612": "localhost:5614", + OrdererEndpointOverrides: map[string]*comm.OrdererEndpoint{ + "localhost:5612": { + Address: "localhost:5614", + }, }, }, deliverClients: map[string]*deliverClient{ @@ -698,7 +700,7 @@ func TestDeliverServiceBadConfig(t *testing.T) { } func TestRetryPolicyOverflow(t *testing.T) { - connFactory := func(channelID string) func(comm.EndpointCriteria) (*grpc.ClientConn, error) { + connFactory := func(channelID string, _ map[string]*comm.OrdererEndpoint) func(comm.EndpointCriteria) (*grpc.ClientConn, error) { return func(_ comm.EndpointCriteria) (*grpc.ClientConn, error) { return nil, errors.New("") } @@ -781,8 +783,10 @@ func TestToEndpointCriteria(t *testing.T) { input: ConnectionCriteria{ Organizations: []string{"foo", "bar"}, OrdererEndpoints: []string{"a", "b", "c"}, - OrdererEndpointOverrides: map[string]string{ - "b": "d", + OrdererEndpointOverrides: map[string]*comm.OrdererEndpoint{ + "b": { + Address: "d", + }, }, }, expectedOut: []comm.EndpointCriteria{ @@ -818,8 +822,10 @@ func TestToEndpointCriteria(t *testing.T) { "foo": {"a", "b"}, "bar": {"c"}, }, - OrdererEndpointOverrides: map[string]string{ - "b": "d", + OrdererEndpointOverrides: map[string]*comm.OrdererEndpoint{ + "b": { + Address: "d", + }, }, }, expectedOut: []comm.EndpointCriteria{ diff --git a/core/peer/config.go b/core/peer/config.go index a6238da6cc6..ec69cc9f996 100644 --- a/core/peer/config.go +++ b/core/peer/config.go @@ -258,22 +258,36 @@ func GetClientCertificate() (tls.Certificate, error) { } type addressOverride struct { - From string `mapstructure:"from"` - To string `mapstructure:"to"` + From string `mapstructure:"from"` + To string `mapstructure:"to"` + CACertsFile string `mapstructure:"caCertsFile"` } -func GetOrdererAddressOverrides() (map[string]string, error) { +type OrdererEndpoint struct { + Address string + PEMs []byte +} + +func GetOrdererAddressOverrides() (map[string]*comm.OrdererEndpoint, error) { var overrides []addressOverride err := viper.UnmarshalKey("peer.deliveryclient.addressOverrides", &overrides) if err != nil { return nil, err } - var overrideMap map[string]string + var overrideMap map[string]*comm.OrdererEndpoint if len(overrides) > 0 { - overrideMap = make(map[string]string) + overrideMap = make(map[string]*comm.OrdererEndpoint) for _, override := range overrides { - overrideMap[override.From] = override.To + pem, err := ioutil.ReadFile(override.CACertsFile) + if err != nil { + return nil, errors.WithMessage(err, fmt.Sprintf("could not read file '%s' specified for caCertsFile of orderer endpoint override from '%s' to '%s'", override.CACertsFile, override.From, override.To)) + } + + overrideMap[override.From] = &comm.OrdererEndpoint{ + Address: override.To, + PEMs: pem, + } } } return overrideMap, nil diff --git a/core/peer/config_test.go b/core/peer/config_test.go index 23678577c60..6d269b9e9a9 100644 --- a/core/peer/config_test.go +++ b/core/peer/config_test.go @@ -327,8 +327,10 @@ func TestGetOrdererAddressOverrides(t *testing.T) { addressOverrides: - from: myaddress to: youraddress + caCertsFile: testdata/Org1-cert.pem - from: myaddress2 - to: youraddress2` + to: youraddress2 + caCertsFile: testdata/Org1-cert.pem` viper.SetConfigType("yaml") err := viper.ReadConfig(bytes.NewBuffer([]byte(conf))) @@ -336,9 +338,41 @@ func TestGetOrdererAddressOverrides(t *testing.T) { t.Fatalf("Failed to read test config: %s", err) } - expected := map[string]string{ - "myaddress": "youraddress", - "myaddress2": "youraddress2", + expected := map[string]*comm.OrdererEndpoint{ + "myaddress": { + Address: "youraddress", + PEMs: []byte(`-----BEGIN CERTIFICATE----- +MIIB8TCCAZegAwIBAgIQU59imQ+xl+FmwuiFyUgFezAKBggqhkjOPQQDAjBYMQsw +CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy +YW5jaXNjbzENMAsGA1UEChMET3JnMTENMAsGA1UEAxMET3JnMTAeFw0xNzA1MDgw +OTMwMzRaFw0yNzA1MDYwOTMwMzRaMFgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpD +YWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRPcmcx +MQ0wCwYDVQQDEwRPcmcxMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFkpP6EqE +87ghFi25UWLvgPatxDiYKYaVSPvpo/XDJ0+9uUmK/C2r5Bvvxx1t8eTROwN77tEK +r+jbJIxX3ZYQMKNDMEEwDgYDVR0PAQH/BAQDAgGmMA8GA1UdJQQIMAYGBFUdJQAw +DwYDVR0TAQH/BAUwAwEB/zANBgNVHQ4EBgQEAQIDBDAKBggqhkjOPQQDAgNIADBF +AiEA1Xkrpq+wrmfVVuY12dJfMQlSx+v0Q3cYce9BE1i2mioCIAzqyduK/lHPI81b +nWiU9JF9dRQ69dEV9dxd/gzamfFU +-----END CERTIFICATE----- +`), + }, + "myaddress2": { + Address: "youraddress2", + PEMs: []byte(`-----BEGIN CERTIFICATE----- +MIIB8TCCAZegAwIBAgIQU59imQ+xl+FmwuiFyUgFezAKBggqhkjOPQQDAjBYMQsw +CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy +YW5jaXNjbzENMAsGA1UEChMET3JnMTENMAsGA1UEAxMET3JnMTAeFw0xNzA1MDgw +OTMwMzRaFw0yNzA1MDYwOTMwMzRaMFgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpD +YWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRPcmcx +MQ0wCwYDVQQDEwRPcmcxMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFkpP6EqE +87ghFi25UWLvgPatxDiYKYaVSPvpo/XDJ0+9uUmK/C2r5Bvvxx1t8eTROwN77tEK +r+jbJIxX3ZYQMKNDMEEwDgYDVR0PAQH/BAQDAgGmMA8GA1UdJQQIMAYGBFUdJQAw +DwYDVR0TAQH/BAUwAwEB/zANBgNVHQ4EBgQEAQIDBDAKBggqhkjOPQQDAgNIADBF +AiEA1Xkrpq+wrmfVVuY12dJfMQlSx+v0Q3cYce9BE1i2mioCIAzqyduK/lHPI81b +nWiU9JF9dRQ69dEV9dxd/gzamfFU +-----END CERTIFICATE----- +`), + }, } overrides, err := GetOrdererAddressOverrides() if err != nil { diff --git a/gossip/service/gossip_service.go b/gossip/service/gossip_service.go index 7649265f9b0..de68c7d8b3c 100644 --- a/gossip/service/gossip_service.go +++ b/gossip/service/gossip_service.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/hyperledger/fabric/common/metrics" + "github.com/hyperledger/fabric/core/comm" "github.com/hyperledger/fabric/core/committer" "github.com/hyperledger/fabric/core/committer/txvalidator" "github.com/hyperledger/fabric/core/common/privdata" @@ -83,7 +84,7 @@ type OrdererAddressConfig struct { Addresses []string AddressesByOrg map[string][]string Organizations []string - AddressOverrides map[string]string + AddressOverrides map[string]*comm.OrdererEndpoint } type privateHandler struct {