Migration from Raft to BFT test (#4561)

* Raft to BFT migration

Signed-off-by: Yacov Manevich <yacov.manevich@ibm.com>

* Raft to BFT migration: functions + tests updates

Signed-off-by: May Rosenbaum <mayro1595@gmail.com>

* Raft to BFT migration: add and edit test cases

Signed-off-by: May Rosenbaum <mayro1595@gmail.com>

* break import cycle to prevent code duplication

Signed-off-by: May Rosenbaum <mayro1595@gmail.com>

---------

Signed-off-by: Yacov Manevich <yacov.manevich@ibm.com>
Signed-off-by: May Rosenbaum <mayro1595@gmail.com>
Co-authored-by: Yacov Manevich <yacov.manevich@ibm.com>

orderer/common/msgprocessor/maintenancefilter.go

@@ -9,10 +9,13 @@ package msgprocessor
 import (
 	"bytes"
 
+	"github.com/hyperledger/fabric/orderer/consensus/smartbft/util"
+
 	"github.com/golang/protobuf/proto"
 	cb "github.com/hyperledger/fabric-protos-go/common"
 	"github.com/hyperledger/fabric-protos-go/orderer"
-	protoetcdraft "github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
+	"github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
+	"github.com/hyperledger/fabric-protos-go/orderer/smartbft"
 	"github.com/hyperledger/fabric/bccsp"
 	"github.com/hyperledger/fabric/common/channelconfig"
 	"github.com/hyperledger/fabric/common/configtx"
@@ -45,10 +48,7 @@ func NewMaintenanceFilter(support MaintenanceFilterSupport, bccsp bccsp.BCCSP) *
 		permittedTargetConsensusTypes: make(map[string]bool),
 		bccsp:                         bccsp,
 	}
-	mf.permittedTargetConsensusTypes["etcdraft"] = true
-	// Until we have a BFT consensus type, we use this for integration testing of consensus-type migration.
-	// Caution: proposing a config block with this type will cause panic.
-	mf.permittedTargetConsensusTypes["testing-only"] = true
+	mf.permittedTargetConsensusTypes["BFT"] = true
 	return mf
 }
 
@@ -119,7 +119,7 @@ func (mf *MaintenanceFilter) inspect(configEnvelope *cb.ConfigEnvelope, ordererC
 	}
 
 	// ConsensusType.Type can only change in maintenance-mode, and only within the set of permitted types.
-	// Note: only solo to etcdraft transitions are supported.
+	// Note: only etcdraft to BFT transitions are supported.
 	if ordererConfig.ConsensusType() != nextOrdererConfig.ConsensusType() {
 		if ordererConfig.ConsensusState() == orderer.ConsensusType_STATE_NORMAL {
 			return errors.Errorf("attempted to change consensus type from %s to %s, but current config ConsensusType.State is not in maintenance mode",
@@ -135,10 +135,20 @@ func (mf *MaintenanceFilter) inspect(configEnvelope *cb.ConfigEnvelope, ordererC
 				ordererConfig.ConsensusType(), nextOrdererConfig.ConsensusType())
 		}
 
-		if nextOrdererConfig.ConsensusType() == "etcdraft" {
-			updatedMetadata := &protoetcdraft.ConfigMetadata{}
+		if nextOrdererConfig.ConsensusType() == "BFT" {
+			updatedMetadata := &smartbft.Options{}
 			if err := proto.Unmarshal(nextOrdererConfig.ConsensusMetadata(), updatedMetadata); err != nil {
-				return errors.Wrap(err, "failed to unmarshal etcdraft metadata configuration")
+				return errors.Wrap(err, "failed to unmarshal BFT metadata configuration")
+			}
+
+			_, err := util.ConfigFromMetadataOptions(1, updatedMetadata)
+			if err != nil {
+				return errors.New("invalid BFT metadata configuration")
+			}
+
+			err = validateBFTConsenterMapping(ordererConfig, nextOrdererConfig)
+			if err != nil {
+				return errors.Wrap(err, "invalid BFT consenter mapping configuration")
 			}
 		}
 
@@ -199,3 +209,38 @@ func (mf *MaintenanceFilter) ensureConsensusTypeChangeOnly(configEnvelope *cb.Co
 
 	return nil
 }
+
+func validateBFTConsenterMapping(currentOrdererConfig channelconfig.Orderer, nextOrdererConfig channelconfig.Orderer) error {
+	// extract raft consenters from consensusTypeValue.metadata
+	raftMetadata := &etcdraft.ConfigMetadata{}
+	proto.Unmarshal(currentOrdererConfig.ConsensusMetadata(), raftMetadata)
+	raftConsenters := raftMetadata.GetConsenters()
+
+	// extract bft consenters
+	bftConsenters := nextOrdererConfig.Consenters()
+
+	if len(bftConsenters) == 0 {
+		return errors.Errorf("Invalid new config: bft consenters are missing")
+	}
+
+	if len(raftConsenters) != len(bftConsenters) {
+		return errors.Errorf("Invalid new config: the number of bft consenters: %d is not equal to the number of raft consenters: %d", len(bftConsenters), len(raftConsenters))
+	}
+
+	for _, raftConsenter := range raftConsenters {
+		flag := false
+		for _, bftConsenter := range bftConsenters {
+			if raftConsenter.Port == bftConsenter.Port && raftConsenter.Host == bftConsenter.Host &&
+				bytes.Equal(raftConsenter.ServerTlsCert, bftConsenter.ServerTlsCert) &&
+				bytes.Equal(raftConsenter.ClientTlsCert, bftConsenter.ClientTlsCert) {
+				flag = true
+				break
+			}
+		}
+		if !flag {
+			return errors.Errorf("No suitable BFT consenter for Raft consenter: %v", raftConsenter)
+		}
+	}
+
+	return nil
+}

orderer/consensus/etcdraft/chain.go

@@ -1115,7 +1115,26 @@ func (c *Chain) commitBlock(block *common.Block) {
 func (c *Chain) detectConfChange(block *common.Block) *MembershipChanges {
 	// If config is targeting THIS channel, inspect consenter set and
 	// propose raft ConfChange if it adds/removes node.
-	configMetadata := c.newConfigMetadata(block)
+	c.logger.Infof("Detected configuration change")
+
+	// if the consensusType is bft, then configMetadata which represents the raft metadata should be nil
+	configMetadata, consensusType := c.newConfigMetadata(block)
+	c.logger.Infof("Detected configuration change: consensusType is: %s, configMetadata is: %v", consensusType, configMetadata)
+
+	if consensusType == nil {
+		c.logger.Infof("ConsensusType is %v", consensusType)
+		return nil
+	}
+
+	if consensusType.Type != "etcdraft" {
+		if consensusType.Type == "BFT" {
+			c.logger.Infof("Detected migration to %s", consensusType.Type)
+			return nil
+		} else {
+			c.logger.Panicf("illegal consensus type detected: %s", consensusType.Type)
+			panic("illegal consensus type detected during conf change")
+		}
+	}
 
 	if configMetadata == nil {
 		return nil
@@ -1425,12 +1444,13 @@ func (c *Chain) getInFlightConfChange() *raftpb.ConfChange {
 }
 
 // newMetadata extract config metadata from the configuration block
-func (c *Chain) newConfigMetadata(block *common.Block) *etcdraft.ConfigMetadata {
-	metadata, err := ConsensusMetadataFromConfigBlock(block)
+func (c *Chain) newConfigMetadata(block *common.Block) (*etcdraft.ConfigMetadata, *orderer.ConsensusType) {
+	c.logger.Infof("Extract config metadata from the configuration block")
+	metadata, consensusType, err := ConsensusMetadataFromConfigBlock(block)
 	if err != nil {
 		c.logger.Panicf("error reading consensus metadata: %s", err)
 	}
-	return metadata
+	return metadata, consensusType
 }
 
 // ValidateConsensusMetadata determines the validity of a
@@ -1446,6 +1466,17 @@ func (c *Chain) ValidateConsensusMetadata(oldOrdererConfig, newOrdererConfig cha
 		return nil
 	}
 
+	if newOrdererConfig.ConsensusType() != "etcdraft" {
+		if newOrdererConfig.ConsensusType() == "BFT" {
+			// This is a migration, so we don't know how to validate this config change.
+			return nil
+		} else {
+			c.logger.Panicf("illegal consensus type detected during consensus metadata validation: %s", newOrdererConfig.ConsensusType())
+			return errors.Errorf("illegal consensus type detected during consensus metadata validation: %s", newOrdererConfig.ConsensusType())
+
+		}
+	}
+
 	if oldOrdererConfig == nil {
 		c.logger.Panic("Programming Error: ValidateConsensusMetadata called with nil old channel config")
 		return nil
@@ -1572,7 +1603,7 @@ func (c *Chain) checkForEvictionNCertRotation(env *common.Envelope) bool {
 		return false
 	}
 
-	configMeta, err := MetadataFromConfigUpdate(configUpdate)
+	configMeta, _, err := MetadataFromConfigUpdate(configUpdate)
 	if err != nil || configMeta == nil {
 		c.logger.Warnf("could not read config metadata: %s", err)
 		return false

orderer/consensus/etcdraft/chain_test.go

@@ -552,11 +552,19 @@ var _ = Describe("Chain", func() {
 								},
 								"ConsensusType": {
 									Version: 4,
+									Value: marshalOrPanic(&orderer.ConsensusType{
+										Type:     "etcdraft",
+										Metadata: []byte{1, 2, 3},
+									}),
 								},
 							}
 							oldValues := map[string]*common.ConfigValue{
 								"ConsensusType": {
 									Version: 4,
+									Value: marshalOrPanic(&orderer.ConsensusType{
+										Type:     "etcdraft",
+										Metadata: []byte{1, 2, 3},
+									}),
 								},
 							}
 							configEnv = newConfigEnv(channelID,
@@ -678,6 +686,7 @@ var _ = Describe("Chain", func() {
 								"ConsensusType": {
 									Version: 1,
 									Value: marshalOrPanic(&orderer.ConsensusType{
+										Type:     "etcdraft",
 										Metadata: marshalOrPanic(consenterMetadata),
 									}),
 								},
@@ -711,6 +720,7 @@ var _ = Describe("Chain", func() {
 								"ConsensusType": {
 									Version: 1,
 									Value: marshalOrPanic(&orderer.ConsensusType{
+										Type:     "etcdraft",
 										Metadata: marshalOrPanic(metadata),
 									}),
 								},
@@ -726,6 +736,35 @@ var _ = Describe("Chain", func() {
 						})
 					})
 				})
+
+				Context("when a type C config update comes", func() {
+					Context("change from raft to bft", func() {
+						// use to prepare the Orderer Values
+						BeforeEach(func() {
+							values := map[string]*common.ConfigValue{
+								"ConsensusType": {
+									Version: 1,
+									Value: marshalOrPanic(&orderer.ConsensusType{
+										Type:     "BFT",
+										Metadata: []byte{1, 2},
+									}),
+								},
+							}
+							configEnv = newConfigEnv(channelID,
+								common.HeaderType_CONFIG,
+								newConfigUpdateEnv(channelID, nil, values))
+							configSeq = 0
+						}) // BeforeEach block
+
+						It("should be able to process config update of type C", func() {
+							err := chain.Configure(configEnv, configSeq)
+							Expect(err).NotTo(HaveOccurred())
+							Expect(fakeFields.fakeConfigProposalsReceived.AddCallCount()).To(Equal(1))
+							Expect(fakeFields.fakeConfigProposalsReceived.AddArgsForCall(0)).To(Equal(float64(1)))
+							Eventually(support.WriteConfigBlockCallCount, LongEventualTimeout).Should(Equal(1))
+						})
+					})
+				})
 			})
 
 			Describe("Crash Fault Tolerance", func() {
@@ -1426,6 +1465,7 @@ var _ = Describe("Chain", func() {
 				"ConsensusType": {
 					Version: 1,
 					Value: marshalOrPanic(&orderer.ConsensusType{
+						Type:     "etcdraft",
 						Metadata: marshalOrPanic(metadata),
 					}),
 				},
@@ -1770,6 +1810,7 @@ var _ = Describe("Chain", func() {
 						"ConsensusType": {
 							Version: 1,
 							Value: marshalOrPanic(&orderer.ConsensusType{
+								Type:     "etcdraft",
 								Metadata: marshalOrPanic(metadata),
 							}),
 						},
@@ -1853,6 +1894,7 @@ var _ = Describe("Chain", func() {
 						"ConsensusType": {
 							Version: 1,
 							Value: marshalOrPanic(&orderer.ConsensusType{
+								Type:     "etcdraft",
 								Metadata: marshalOrPanic(metadata),
 							}),
 						},
@@ -1893,6 +1935,7 @@ var _ = Describe("Chain", func() {
 						"ConsensusType": {
 							Version: 1,
 							Value: marshalOrPanic(&orderer.ConsensusType{
+								Type:     "etcdraft",
 								Metadata: marshalOrPanic(metadata),
 							}),
 						},
@@ -1935,6 +1978,7 @@ var _ = Describe("Chain", func() {
 							"ConsensusType": {
 								Version: 1,
 								Value: marshalOrPanic(&orderer.ConsensusType{
+									Type:     "etcdraft",
 									Metadata: marshalOrPanic(metadata),
 								}),
 							},

orderer/consensus/etcdraft/util.go

@@ -84,22 +84,27 @@ func MetadataHasDuplication(md *etcdraft.ConfigMetadata) error {
 }
 
 // MetadataFromConfigValue reads and translates configuration updates from config value into raft metadata
-func MetadataFromConfigValue(configValue *common.ConfigValue) (*etcdraft.ConfigMetadata, error) {
+// In case consensus type is changed to BFT the raft metadata will be nil
+func MetadataFromConfigValue(configValue *common.ConfigValue) (*etcdraft.ConfigMetadata, *orderer.ConsensusType, error) {
 	consensusTypeValue := &orderer.ConsensusType{}
 	if err := proto.Unmarshal(configValue.Value, consensusTypeValue); err != nil {
-		return nil, errors.Wrap(err, "failed to unmarshal consensusType config update")
+		return nil, nil, errors.Wrap(err, "failed to unmarshal consensusType config update")
+	}
+
+	if consensusTypeValue.Type != "etcdraft" {
+		return nil, consensusTypeValue, nil
 	}
 
 	updatedMetadata := &etcdraft.ConfigMetadata{}
 	if err := proto.Unmarshal(consensusTypeValue.Metadata, updatedMetadata); err != nil {
-		return nil, errors.Wrap(err, "failed to unmarshal updated (new) etcdraft metadata configuration")
+		return nil, nil, errors.Wrap(err, "failed to unmarshal updated (new) etcdraft metadata configuration")
 	}
 
-	return updatedMetadata, nil
+	return updatedMetadata, consensusTypeValue, nil
 }
 
 // MetadataFromConfigUpdate extracts consensus metadata from config update
-func MetadataFromConfigUpdate(update *common.ConfigUpdate) (*etcdraft.ConfigMetadata, error) {
+func MetadataFromConfigUpdate(update *common.ConfigUpdate) (*etcdraft.ConfigMetadata, *orderer.ConsensusType, error) {
 	var baseVersion uint64
 	if update.ReadSet != nil && update.ReadSet.Groups != nil {
 		if ordererConfigGroup, ok := update.ReadSet.Groups["Orderer"]; ok {
@@ -115,13 +120,13 @@ func MetadataFromConfigUpdate(update *common.ConfigUpdate) (*etcdraft.ConfigMeta
 				if baseVersion == val.Version {
 					// Only if the version in the write set differs from the read-set
 					// should we consider this to be an update to the consensus type
-					return nil, nil
+					return nil, nil, nil
 				}
 				return MetadataFromConfigValue(val)
 			}
 		}
 	}
-	return nil, nil
+	return nil, nil, nil
 }
 
 // ConfigChannelHeader expects a config block and returns the header type
@@ -168,28 +173,28 @@ func ConfigEnvelopeFromBlock(block *common.Block) (*common.Envelope, error) {
 }
 
 // ConsensusMetadataFromConfigBlock reads consensus metadata updates from the configuration block
-func ConsensusMetadataFromConfigBlock(block *common.Block) (*etcdraft.ConfigMetadata, error) {
+func ConsensusMetadataFromConfigBlock(block *common.Block) (*etcdraft.ConfigMetadata, *orderer.ConsensusType, error) {
 	if block == nil {
-		return nil, errors.New("nil block")
+		return nil, nil, errors.New("nil block")
 	}
 
 	if !protoutil.IsConfigBlock(block) {
-		return nil, errors.New("not a config block")
+		return nil, nil, errors.New("not a config block")
 	}
 
 	configEnvelope, err := ConfigEnvelopeFromBlock(block)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannot read config update")
+		return nil, nil, errors.Wrap(err, "cannot read config update")
 	}
 
 	payload, err := protoutil.UnmarshalPayload(configEnvelope.Payload)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to extract payload from config envelope")
+		return nil, nil, errors.Wrap(err, "failed to extract payload from config envelope")
 	}
 	// get config update
 	configUpdate, err := configtx.UnmarshalConfigUpdateFromPayload(payload)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not read config update")
+		return nil, nil, errors.Wrap(err, "could not read config update")
 	}
 
 	return MetadataFromConfigUpdate(configUpdate)

orderer/consensus/etcdraft/util_test.go

@@ -17,6 +17,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hyperledger/fabric-protos-go/orderer"
+
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric-protos-go/common"
 	etcdraftproto "github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
@@ -410,3 +412,20 @@ func TestVerifyConfigMetadata(t *testing.T) {
 		require.Nil(t, VerifyConfigMetadata(metadataWithExpiredConsenter, goodVerifyingOpts))
 	})
 }
+
+// This test checks that MetadataFromConfigValue, which reads and translates configuration updates from config value,
+// returns an empty raft metadata when the consensus type is BFT.
+func TestMetadataFromConfigValue(t *testing.T) {
+	configValue := &common.ConfigValue{
+		Value: protoutil.MarshalOrPanic(&orderer.ConsensusType{
+			Type:     "BFT",
+			Metadata: []byte{1, 2},
+		}),
+	}
+
+	metadata, consensusType, err := MetadataFromConfigValue(configValue)
+	require.Nil(t, metadata)
+	require.Nil(t, err)
+	require.NotNil(t, consensusType)
+	require.Equal(t, consensusType.Type, "BFT")
+}