diff --git a/integration/raft/migration_test.go b/integration/raft/migration_test.go index c489b38f394..553713cb63e 100644 --- a/integration/raft/migration_test.go +++ b/integration/raft/migration_test.go @@ -23,7 +23,6 @@ import ( "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-protos-go/common" protosorderer "github.com/hyperledger/fabric-protos-go/orderer" - protosraft "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" "github.com/hyperledger/fabric/integration/channelparticipation" "github.com/hyperledger/fabric/integration/nwo" "github.com/hyperledger/fabric/integration/nwo/commands" @@ -75,6 +74,10 @@ var _ = Describe("ConsensusTypeMigration", func() { _ = os.RemoveAll(testDir) }) + // This test executes the migration on an etcdraft based system. + // The migration includes change to maintenance mode, config updates needed for migration to a bft based system and + // exiting maintenance mode back to the normal state. + // This test restarts the orderers and ensures they operate under the new configuration. Describe("Raft to BFT migration", func() { It("migrates from Raft to BFT", func() { networkConfig := nwo.MultiNodeEtcdRaft() @@ -126,39 +129,25 @@ var _ = Describe("ConsensusTypeMigration", func() { By("Change to maintenance mode") peer := network.Peer("Org1", "peer0") - orderer := network.Orderer("orderer1") - - addBFTInConfigTree(network, peer, orderer, "testchannel") By("Config update on standard channel, State=MAINTENANCE, enter maintenance-mode") - config, updatedConfig := prepareTransition(network, peer, orderer, "testchannel", + config, updatedConfig := prepareTransition(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, "etcdraft", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) - nwo.UpdateOrdererConfig(network, orderer, "testchannel", config, updatedConfig, peer, orderer) - - bftMetadata := protoutil.MarshalOrPanic(&smartbft.Options{ - RequestBatchMaxCount: types.DefaultConfig.RequestBatchMaxCount, - RequestBatchMaxBytes: types.DefaultConfig.RequestBatchMaxBytes, - RequestBatchMaxInterval: types.DefaultConfig.RequestBatchMaxInterval.String(), - IncomingMessageBufferSize: types.DefaultConfig.IncomingMessageBufferSize, - RequestPoolSize: types.DefaultConfig.RequestPoolSize, - RequestForwardTimeout: types.DefaultConfig.RequestForwardTimeout.String(), - RequestComplainTimeout: types.DefaultConfig.RequestComplainTimeout.String(), - RequestAutoRemoveTimeout: types.DefaultConfig.RequestAutoRemoveTimeout.String(), - ViewChangeResendInterval: types.DefaultConfig.ViewChangeResendInterval.String(), - ViewChangeTimeout: types.DefaultConfig.ViewChangeTimeout.String(), - LeaderHeartbeatTimeout: types.DefaultConfig.LeaderHeartbeatTimeout.String(), - LeaderHeartbeatCount: types.DefaultConfig.LeaderHeartbeatCount, - CollectTimeout: types.DefaultConfig.CollectTimeout.String(), - SyncOnStart: types.DefaultConfig.SyncOnStart, - SpeedUpViewChange: types.DefaultConfig.SpeedUpViewChange, - }) - config, updatedConfig = prepareTransition(network, peer, orderer, "testchannel", + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) + + By("Config updates: migration from Raft to BFT") + + bftMetadata := protoutil.MarshalOrPanic(prepareBftMetadata()) + + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, "BFT", bftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) - nwo.UpdateOrdererConfig(network, orderer, "testchannel", config, updatedConfig, peer, orderer) + currentBlockNumber := nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) - time.Sleep(time.Second) // TODO: check block was committed in all orderers + // check block was committed in all orderers + assertBlockReceptionInAllOrderers(network.Orderers[1:], peer, network, "testchannel", currentBlockNumber) for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc, o4Proc} { if oProc != nil { @@ -169,18 +158,17 @@ var _ = Describe("ConsensusTypeMigration", func() { runOrderers() - // TODO: By("Waiting for followers to see the leader, again") Eventually(o2Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Message from 1 channel=testchannel")) Eventually(o3Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Message from 1 channel=testchannel")) Eventually(o4Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Message from 1 channel=testchannel")) // Exit maintenance mode - - config, updatedConfig = prepareTransition(network, peer, orderer, "testchannel", + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", "BFT", protosorderer.ConsensusType_STATE_MAINTENANCE, "BFT", bftMetadata, protosorderer.ConsensusType_STATE_NORMAL) - nwo.UpdateOrdererConfig(network, orderer, "testchannel", config, updatedConfig, peer, orderer) + currentBlockNumber = nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) // Now, run a transaction to ensure BFT works. env = CreateBroadcastEnvelope(network, o1, "testchannel", []byte("foo")) @@ -188,379 +176,246 @@ var _ = Describe("ConsensusTypeMigration", func() { Expect(err).NotTo(HaveOccurred()) Expect(resp.Status).To(Equal(common.Status_SUCCESS)) - time.Sleep(time.Second * 5) - - // TODO: check block was successfully committed in all orderers + // check block was successfully committed in all orderers + assertBlockReceptionInAllOrderers(network.Orderers[1:], peer, network, "testchannel", currentBlockNumber) }) }) - // These tests execute the migration config updates on an etcdraft based system, but do not restart the orderer - // to a "future-type" consensus-type that currently does not exist. However, these tests try to maintain as much as - // possible the testing infrastructure for consensus-type migration that existed when "solo" and "kafka" were still - // supported. When a future type that can be migrated to from raft becomes available, some test-cases within this - // suite will need to be completed and revised. - Describe("Raft to future-type migration", func() { + Describe("Enforcing config updates during Raft to BFT migration", func() { var ( - orderer *nwo.Orderer - peer *nwo.Peer - channel1, channel2 string + o1 *nwo.Orderer + o2 *nwo.Orderer + o3 *nwo.Orderer + o4 *nwo.Orderer + peer *nwo.Peer ) BeforeEach(func() { - network = nwo.New(nwo.MultiChannelEtcdRaft(), testDir, client, StartPort(), components) + networkConfig := nwo.MultiNodeEtcdRaft() + networkConfig.Orderers = append(networkConfig.Orderers, &nwo.Orderer{Name: "orderer4", Organization: "OrdererOrg"}) + networkConfig.Profiles[0].Orderers = []string{"orderer1", "orderer2", "orderer3", "orderer4"} + + network = nwo.New(networkConfig, testDir, client, StartPort(), components) + + o1, o2, o3, o4 = network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3"), network.Orderer("orderer4") network.GenerateConfigTree() network.Bootstrap() - orderer = network.Orderer("orderer") - peer = network.Peer("Org1", "peer0") + runOrderers := func() { + o1Runner = network.OrdererRunner(o1) + o2Runner = network.OrdererRunner(o2) + o3Runner = network.OrdererRunner(o3) + o4Runner = network.OrdererRunner(o4) + + o1Runner.Command.Env = append(o1Runner.Command.Env, "FABRIC_LOGGING_SPEC=orderer.common.cluster=debug:orderer.consensus.smartbft=debug:policies.ImplicitOrderer=debug") + o2Runner.Command.Env = append(o2Runner.Command.Env, "FABRIC_LOGGING_SPEC=orderer.common.cluster=debug:orderer.consensus.smartbft=debug:policies.ImplicitOrderer=debug") + o3Runner.Command.Env = append(o3Runner.Command.Env, "FABRIC_LOGGING_SPEC=orderer.common.cluster=debug:orderer.consensus.smartbft=debug:policies.ImplicitOrderer=debug") + o4Runner.Command.Env = append(o4Runner.Command.Env, "FABRIC_LOGGING_SPEC=orderer.common.cluster=debug:orderer.consensus.smartbft=debug:policies.ImplicitOrderer=debug") + + o1Proc = ifrit.Invoke(o1Runner) + o2Proc = ifrit.Invoke(o2Runner) + o3Proc = ifrit.Invoke(o3Runner) + o4Proc = ifrit.Invoke(o4Runner) + + Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) + Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) + Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) + Eventually(o4Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) + } - channel1 = "testchannel" - channel2 = "testchannel2" + runOrderers() - o1Runner = network.OrdererRunner(orderer) - o1Proc = ifrit.Invoke(o1Runner) - Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) + channelparticipation.JoinOrderersAppChannelCluster(network, "testchannel", o1, o2, o3, o4) + FindLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner, o4Runner}) - By("Create & join first channel") - channelparticipation.JoinOrdererAppChannel(network, channel1, orderer, o1Runner) + peer = network.Peer("Org1", "peer0") }) // This test executes the "green path" migration config updates on an etcdraft based system, on a standard // channel, and verifies that these config updates have the desired effect. - // // The green path is entering maintenance mode, and then changing the consensus type. // In maintenance mode we check that normal transactions are blocked. - // // We also check that after entering maintenance mode, we can exit it without making any changes - the "abort path". It("executes raft2future green path", func() { // === The abort path ====================================================================================== // === Step 1: Config update on standard channel, MAINTENANCE === By("1) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode") - config, updatedConfig := prepareTransition(network, peer, orderer, channel1, + config, updatedConfig := prepareTransition(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, "etcdraft", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) By("1) Verify: standard channel config changed") - std1EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) + std1EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") Expect(std1EntryBlockNum).ToNot(Equal(0)) - config = nwo.GetConfig(network, peer, orderer, channel1) + config = nwo.GetConfig(network, peer, o1, "testchannel") consensusTypeValue := extractOrdererConsensusType(config) validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) By("1) Verify: Normal TX's on standard channel are blocked") - assertTxFailed(network, orderer, channel1) + assertTxFailed(network, o1, "testchannel") // In maintenance mode deliver requests are open to those entities that satisfy the /Channel/Orderer/Readers policy By("1) Verify: delivery request from peer is blocked") - err := checkPeerDeliverRequest(orderer, peer, network, channel1) + err := checkPeerDeliverRequest(o1, peer, network, "testchannel") Expect(err).To(MatchError(errors.New("FORBIDDEN"))) - // === Step 2: Create a new channel - By("2) Create & join second channel") - channelparticipation.JoinOrdererAppChannel(network, channel2, orderer, o1Runner) - assertBlockCreation(network, orderer, peer, channel2, 1) - - By("2) Verify: delivery request from peer is not blocked on new channel") - err = checkPeerDeliverRequest(orderer, peer, network, channel2) - Expect(err).NotTo(HaveOccurred()) - - // === Step 3: config update on standard channel, State=NORMAL, abort === - By("3) Config update on standard channel, State=NORMAL, exit maintenance-mode - abort path") - config, updatedConfig = prepareTransition(network, peer, orderer, channel1, + // === Step 2: config update on standard channel, State=NORMAL, abort === + By("2) Config update on standard channel, State=NORMAL, exit maintenance-mode - abort path") + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, "etcdraft", nil, protosorderer.ConsensusType_STATE_NORMAL) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) - By("3) Verify: standard channel config changed") - std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) + By("2) Verify: standard channel config changed") + std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") Expect(std1BlockNum).To(Equal(std1EntryBlockNum + 1)) - By("3) Verify: standard channel delivery requests from peer unblocked") - err = checkPeerDeliverRequest(orderer, peer, network, channel1) + By("2) Verify: standard channel delivery requests from peer unblocked") + err = checkPeerDeliverRequest(o1, peer, network, "testchannel") Expect(err).NotTo(HaveOccurred()) - By("3) Verify: Normal TX's on standard channel are permitted again") - assertBlockCreation(network, orderer, nil, channel1, 3) + By("2) Verify: Normal TX's on standard channel are permitted again") + assertBlockCreation(network, o1, nil, "testchannel", std1EntryBlockNum+2) // === The green path ====================================================================================== - // === Step 4: Config update on standard channel1, MAINTENANCE, again === - By("4) Config update on standard channel1, State=MAINTENANCE, enter maintenance-mode again") - config, updatedConfig = prepareTransition(network, peer, orderer, channel1, + //=== Step 3: Config update on standard channel, MAINTENANCE, again === + By("3) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode again") + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, "etcdraft", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) - By("4) Verify: standard channel config changed") - std1EntryBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) - Expect(std1EntryBlockNum).ToNot(Equal(0)) + By("3) Verify: standard channel config changed") + std1EntryBlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") + Expect(std1EntryBlockNum).To(Equal(uint64(4))) - config = nwo.GetConfig(network, peer, orderer, channel1) + config = nwo.GetConfig(network, peer, o1, "testchannel") consensusTypeValue = extractOrdererConsensusType(config) validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) - By("4) Verify: delivery request from peer is blocked") - err = checkPeerDeliverRequest(orderer, peer, network, channel1) + By("3) Verify: delivery request from peer is blocked") + err = checkPeerDeliverRequest(o1, peer, network, "testchannel") Expect(err).To(MatchError(errors.New("FORBIDDEN"))) - By("4) Verify: Normal TX's on standard channel are blocked") - assertTxFailed(network, orderer, channel1) + By("3) Verify: Normal TX's on standard channel are blocked") + assertTxFailed(network, o1, "testchannel") - // === Step 5: Config update on standard channel2, MAINTENANCE === - By("5) Config update on standard channel2, State=MAINTENANCE, enter maintenance-mode again") - config, updatedConfig = prepareTransition(network, peer, orderer, channel2, - "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, - "etcdraft", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) - nwo.UpdateOrdererConfig(network, orderer, channel2, config, updatedConfig, peer, orderer) + // === Step 4: config update on standard channel, State=MAINTENANCE, type=BFT === + bftMetadata := protoutil.MarshalOrPanic(prepareBftMetadata()) - By("5) Verify: standard channel config changed") - std2EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2) - Expect(std2EntryBlockNum).ToNot(Equal(0)) - - config = nwo.GetConfig(network, peer, orderer, channel2) - consensusTypeValue = extractOrdererConsensusType(config) - validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) + By("4) config update on standard channel, State=MAINTENANCE, type=BFT") + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", + "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, + "BFT", bftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) - By("5) Verify: delivery request from peer is blocked") - err = checkPeerDeliverRequest(orderer, peer, network, channel2) - Expect(err).To(MatchError(errors.New("FORBIDDEN"))) + By("4) Verify: standard channel config changed") + std1EntryBlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") + Expect(std1EntryBlockNum).To(Equal(uint64(5))) - By("5) Verify: Normal TX's on standard channel are blocked") - assertTxFailed(network, orderer, channel2) + config = nwo.GetConfig(network, peer, o1, "testchannel") + consensusTypeValue = extractOrdererConsensusType(config) + validateConsensusTypeValue(consensusTypeValue, "BFT", protosorderer.ConsensusType_STATE_MAINTENANCE) - // Note: - // The following testing steps should be completed once we have a consensus-type ("future-type") that can be - // migrated to from etcdraft. + By("4) Verify: validate consensus type value in all orderers") + // orderer2 + config = nwo.GetConfig(network, peer, o2, "testchannel") + consensusTypeValue = extractOrdererConsensusType(config) + validateConsensusTypeValue(consensusTypeValue, "BFT", protosorderer.ConsensusType_STATE_MAINTENANCE) - // === Step 6: config update on standard channel1, State=MAINTENANCE, type=future-type === + // orderer3 + config = nwo.GetConfig(network, peer, o3, "testchannel") + consensusTypeValue = extractOrdererConsensusType(config) + validateConsensusTypeValue(consensusTypeValue, "BFT", protosorderer.ConsensusType_STATE_MAINTENANCE) - // === Step 7: config update on standard channel2, State=MAINTENANCE, type=future-type === + // orderer4 + config = nwo.GetConfig(network, peer, o4, "testchannel") + consensusTypeValue = extractOrdererConsensusType(config) + validateConsensusTypeValue(consensusTypeValue, "BFT", protosorderer.ConsensusType_STATE_MAINTENANCE) }) // This test executes the migration flow and checks that forbidden transitions are rejected. // These transitions are enforced by the maintenance filter: // - Entry to & exit from maintenance mode can only change ConsensusType.State. - // - In maintenance mode one can only change ConsensusType.Type & ConsensusType.Metadata. - // - ConsensusType.Type can only change from "etcdraft" to "future-type" (to be replaced by future consensus - // protocol), and only in maintenance mode. + // - In maintenance mode one can only change ConsensusType.Type & ConsensusType.Metadata & Orderers.ConsenterMapping. + // - ConsensusType.Type can only change from "etcdraft" to "BFT", and only in maintenance mode. It("executes raft2future forbidden transitions", func() { // === Step 1: === By("1) Config update on standard channel, changing both ConsensusType State & Type is forbidden") - assertTransitionFailed(network, peer, orderer, channel1, + assertTransitionFailed(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, - "testing-only", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) + "BFT", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) // === Step 2: === By("2) Config update on standard channel, both ConsensusType State & some other value is forbidden") - config, updatedConfig := prepareTransition(network, peer, orderer, channel1, + config, updatedConfig := prepareTransition(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, "etcdraft", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) updateConfigWithBatchTimeout(updatedConfig) - updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer) + updateOrdererConfigFailed(network, o1, "testchannel", config, updatedConfig, peer, o1) // === Step 3: === By("3) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode") - config, updatedConfig = prepareTransition(network, peer, orderer, channel1, + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, "etcdraft", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) By("3) Verify: standard channel config changed") - std1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) + std1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") Expect(std1StartBlockNum).ToNot(Equal(0)) - config = nwo.GetConfig(network, peer, orderer, channel1) + config = nwo.GetConfig(network, peer, o1, "testchannel") consensusTypeValue := extractOrdererConsensusType(config) validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) // === Step 4: === By("4) Config update on standard channel, change ConsensusType.Type to unsupported type, forbidden") - assertTransitionFailed(network, peer, orderer, channel1, + assertTransitionFailed(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, "hesse", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) // === Step 5: === By("5) Config update on standard channel, change ConsensusType.Type and State, forbidden") - assertTransitionFailed(network, peer, orderer, channel1, + assertTransitionFailed(network, peer, o1, "testchannel", "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, - "testing-only", nil, protosorderer.ConsensusType_STATE_NORMAL) - - // Note: - // The following testing steps should be completed once we have a consensus-type ("future-type" that can be - // migrated to from etcdraft. + "BFT", nil, protosorderer.ConsensusType_STATE_NORMAL) // === Step 6: Config update on standard channel, changing both ConsensusType.Type and other value is permitted === By("6) changing both ConsensusType.Type and other value is permitted") - // Change consensus type and batch-timeout, for example + // Change consensus type and batch-timeout + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", + "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, + "BFT", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) + updateConfigWithBatchTimeout(updatedConfig) + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) + + By("6) Verify: standard channel config changed") + std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") + Expect(std1BlockNum).To(Equal(std1StartBlockNum + 1)) + config = nwo.GetConfig(network, peer, o1, "testchannel") + consensusTypeValue = extractOrdererConsensusType(config) + validateConsensusTypeValue(consensusTypeValue, "BFT", protosorderer.ConsensusType_STATE_MAINTENANCE) // === Step 7: === By("7) Config update on standard channel, changing value other than ConsensusType.Type is permitted") - config = nwo.GetConfig(network, peer, orderer, channel1) - consensusTypeValue = extractOrdererConsensusType(config) - validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) updatedConfig = proto.Clone(config).(*common.Config) updateConfigWithBatchTimeout(updatedConfig) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) + nwo.UpdateOrdererConfig(network, o1, "testchannel", config, updatedConfig, peer, o1) By("7) Verify: standard channel config changed") - std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) - Expect(std1BlockNum).To(Equal(std1StartBlockNum + 1)) + std1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, "testchannel") + Expect(std1BlockNum).To(Equal(std1StartBlockNum + 2)) // === Step 8: === By("8) Config update on standard channel, both ConsensusType State & some other value is forbidden") - config, updatedConfig = prepareTransition(network, peer, orderer, channel1, - "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, - "etcdraft", nil, protosorderer.ConsensusType_STATE_NORMAL) - updateConfigWithBatchTimeout(updatedConfig) - updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer) - }) - - // Note: - // Instead of booting to a future-type which does not exist yet, we change some other config value in - // maintenance mode, reboot, and exit maintenance mode. Once we have a future consensus-type that can be - // migrated to from raft is available, this test should be completed. - It("executes bootstrap to future-type - single node", func() { - // === Step 1: Config update on standard channel, MAINTENANCE === - By("1) Config update on standard channel, State=MAINTENANCE") - config, updatedConfig := prepareTransition(network, peer, orderer, channel1, - "etcdraft", protosorderer.ConsensusType_STATE_NORMAL, - "etcdraft", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) - - By("1) Verify: standard channel config changed") - chan1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) - Expect(chan1StartBlockNum).ToNot(Equal(0)) - - config = nwo.GetConfig(network, peer, orderer, channel1) - consensusTypeValue := extractOrdererConsensusType(config) - validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) - - By("1) Verify: Normal TX's on standard channel are blocked") - assertTxFailed(network, orderer, channel1) - - // In maintenance mode deliver requests are open to those entities that satisfy the /Channel/Orderer/Readers policy - By("1) Verify: delivery request from peer is blocked") - err := checkPeerDeliverRequest(orderer, peer, network, channel1) - Expect(err).To(MatchError(errors.New("FORBIDDEN"))) - - // === Step 2: config update on standard channel, State=MAINTENANCE, type=etcdraft, (in-lieu of future-type) other value changed === - By("2) Config update on standard channel, State=MAINTENANCE, type=etcdraft, (in-lieu of future-type) other value changed") - config = nwo.GetConfig(network, peer, orderer, channel1) - consensusTypeValue = extractOrdererConsensusType(config) - validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) - updatedConfig = proto.Clone(config).(*common.Config) + config, updatedConfig = prepareTransition(network, peer, o1, "testchannel", + "BFT", protosorderer.ConsensusType_STATE_MAINTENANCE, + "BFT", nil, protosorderer.ConsensusType_STATE_NORMAL) updateConfigWithBatchTimeout(updatedConfig) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) - - By("2) Verify: standard channel config changed") - chan1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) - Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 1)) - - // === Step 3: kill === - By("3) killing orderer1") - o1Proc.Signal(syscall.SIGKILL) - Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137"))) - - // === Step 4: restart === - By("4) restarting orderer1") - network.Consensus.Type = "etcdraft" // Note: change to future-type - - o1Runner = network.OrdererRunner(orderer) - o1Proc = ifrit.Invoke(o1Runner) - - Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) - - assertBlockReception( - map[string]int{ - channel1: int(chan1BlockNum), - }, - []*nwo.Orderer{orderer}, - peer, - network, - ) - - Eventually(o1Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> ")) - Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) - - By("5) Release - executing config transaction on standard channel with restarted orderer") - config, updatedConfig = prepareTransition(network, peer, orderer, channel1, - "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, - "etcdraft", nil, protosorderer.ConsensusType_STATE_NORMAL) - nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) - - By("6) Verify: standard channel config changed") - chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) - Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 2)) - - By("7) Executing transaction on standard channel with restarted orderer") - assertBlockCreation(network, orderer, peer, channel1, chan1StartBlockNum+3) - assertBlockCreation(network, orderer, nil, channel1, chan1StartBlockNum+4) - - By("8) Create new channel, executing transaction with restarted orderer") - channelparticipation.JoinOrdererAppChannel(network, channel2, orderer, o1Runner) - - assertBlockCreation(network, orderer, peer, channel2, 1) - assertBlockCreation(network, orderer, nil, channel2, 2) - - By("9) Extending the network configuration to add a new orderer") - // Add another orderer - orderer2 := &nwo.Orderer{ - Name: "orderer2", - Organization: "OrdererOrg", - } - ports := nwo.Ports{} - for _, portName := range nwo.OrdererPortNames() { - ports[portName] = network.ReservePort() - } - network.PortsByOrdererID[orderer2.ID()] = ports - network.Orderers = append(network.Orderers, orderer2) - network.GenerateOrdererConfig(orderer2) - extendNetwork(network) - - secondOrdererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(orderer2), "server.crt") - secondOrdererCertificate, err := os.ReadFile(secondOrdererCertificatePath) - Expect(err).NotTo(HaveOccurred()) - - By("10) Adding the second orderer to second channel") - addConsenter(network, peer, orderer, channel2, protosraft.Consenter{ - ServerTlsCert: secondOrdererCertificate, - ClientTlsCert: secondOrdererCertificate, - Host: "127.0.0.1", - Port: uint32(network.OrdererPort(orderer2, nwo.ClusterPort)), - }) - - By("11) Obtaining the last config block from the orderer") - configBlock := nwo.GetConfigBlock(network, peer, orderer, channel2) - err = os.WriteFile(filepath.Join(testDir, "channel2_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644) - Expect(err).NotTo(HaveOccurred()) - - By("12) Waiting for the existing orderer to relinquish its leadership") - Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("1 stepped down to follower since quorum is not active")) - Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("No leader is present, cluster size is 2")) - - By("13) Launching the second orderer") - o2Runner = network.OrdererRunner(orderer2) - o2Proc = ifrit.Invoke(o2Runner) - Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) - - By("14) Joining the second orderer to channel2") - expectedChannelInfo := channelparticipation.ChannelInfo{ - Name: channel2, - URL: fmt.Sprintf("/participation/v1/channels/%s", channel2), - Status: "onboarding", - ConsensusRelation: "consenter", - Height: 0, - } - channelparticipation.Join(network, orderer2, channel2, configBlock, expectedChannelInfo) - Eventually(o2Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> ")) - - assertBlockReception(map[string]int{ - channel2: int(nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2)), - }, []*nwo.Orderer{orderer2}, peer, network) - - By("15) Executing transaction against second orderer on channel2") - assertBlockCreation(network, orderer2, nil, channel2, 3) + updateOrdererConfigFailed(network, o1, "testchannel", config, updatedConfig, peer, o1) }) }) }) @@ -651,6 +506,9 @@ func prepareTransition( consensusTypeValue := extractOrdererConsensusType(current) validateConsensusTypeValue(consensusTypeValue, fromConsensusType, fromMigState) updateConfigWithConsensusType(toConsensusType, toConsensusMetadata, toMigState, updated, consensusTypeValue) + if toConsensusType == "BFT" { + updateBFTOrderersConfig(network, updated) + } return current, updated } @@ -730,23 +588,17 @@ func createDeliverEnvelope(n *nwo.Network, signer *nwo.SigningIdentity, blkNum u return env } -func addBFTInConfigTree(network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string) { - config := nwo.GetConfig(network, peer, orderer, channel) - - updatedConfig := proto.Clone(config).(*common.Config) - +func updateBFTOrderersConfig(network *nwo.Network, config *common.Config) { orderersVal := &common.Orderers{ ConsenterMapping: computeConsenterMappings(network), } - policies.EncodeBFTBlockVerificationPolicy(orderersVal.ConsenterMapping, updatedConfig.ChannelGroup.Groups["Orderer"]) + policies.EncodeBFTBlockVerificationPolicy(orderersVal.ConsenterMapping, config.ChannelGroup.Groups["Orderer"]) - updatedConfig.ChannelGroup.Groups["Orderer"].Values["Orderers"] = &common.ConfigValue{ + config.ChannelGroup.Groups["Orderer"].Values["Orderers"] = &common.ConfigValue{ Value: protoutil.MarshalOrPanic(orderersVal), ModPolicy: "/Channel/Orderer/Admins", } - - nwo.UpdateOrdererConfig(network, orderer, channel, config, updatedConfig, peer, orderer) } func computeConsenterMappings(network *nwo.Network) []*common.Consenter { @@ -798,3 +650,37 @@ func CreateBroadcastEnvelope(n *nwo.Network, entity interface{}, channel string, return env } + +// assertBlockReception asserts that the given orderers have the expected +// newest block number for the specified channels +func assertBlockReceptionInAllOrderers(orderers []*nwo.Orderer, peer *nwo.Peer, network *nwo.Network, channelId string, currentBlockNumber uint64) { + for _, orderer := range orderers { + ccb := func() uint64 { + return nwo.CurrentConfigBlockNumber(network, peer, orderer, channelId) + } + Eventually(ccb, network.EventuallyTimeout).Should(BeNumerically(">", currentBlockNumber)) + + } +} + +// prepareBftMetadata prepare the bft consensusType.Metadata +func prepareBftMetadata() *smartbft.Options { + bftMetadata := &smartbft.Options{ + RequestBatchMaxCount: types.DefaultConfig.RequestBatchMaxCount, + RequestBatchMaxBytes: types.DefaultConfig.RequestBatchMaxBytes, + RequestBatchMaxInterval: types.DefaultConfig.RequestBatchMaxInterval.String(), + IncomingMessageBufferSize: types.DefaultConfig.IncomingMessageBufferSize, + RequestPoolSize: types.DefaultConfig.RequestPoolSize, + RequestForwardTimeout: types.DefaultConfig.RequestForwardTimeout.String(), + RequestComplainTimeout: types.DefaultConfig.RequestComplainTimeout.String(), + RequestAutoRemoveTimeout: types.DefaultConfig.RequestAutoRemoveTimeout.String(), + ViewChangeResendInterval: types.DefaultConfig.ViewChangeResendInterval.String(), + ViewChangeTimeout: types.DefaultConfig.ViewChangeTimeout.String(), + LeaderHeartbeatTimeout: types.DefaultConfig.LeaderHeartbeatTimeout.String(), + LeaderHeartbeatCount: types.DefaultConfig.LeaderHeartbeatCount, + CollectTimeout: types.DefaultConfig.CollectTimeout.String(), + SyncOnStart: types.DefaultConfig.SyncOnStart, + SpeedUpViewChange: types.DefaultConfig.SpeedUpViewChange, + } + return bftMetadata +} diff --git a/orderer/consensus/etcdraft/chain.go b/orderer/consensus/etcdraft/chain.go index 5fd1db3c21e..7dd4da6ce7a 100644 --- a/orderer/consensus/etcdraft/chain.go +++ b/orderer/consensus/etcdraft/chain.go @@ -1115,14 +1115,23 @@ func (c *Chain) commitBlock(block *common.Block) { func (c *Chain) detectConfChange(block *common.Block) *MembershipChanges { // If config is targeting THIS channel, inspect consenter set and // propose raft ConfChange if it adds/removes node. + c.logger.Infof("Detected configuration change") + + // if the consensusType is bft, then configMetadata which represents the raft metadata should be nil configMetadata, consensusType := c.newConfigMetadata(block) + c.logger.Infof("Detected configuration change: consensusType is: %s, configMetadata is: %v", consensusType.Type, configMetadata) - if configMetadata == nil { - return nil + if consensusType.Type != "etcdraft" { + if consensusType.Type == "BFT" { + c.logger.Infof("Detected migration to %s", consensusType.Type) + return nil + } else { + c.logger.Panicf("illegal consensus type detected: %s", consensusType.Type) + panic("illegal consensus type detected during conf change") + } } - if consensusType.Type != "etcdraft" { - c.logger.Infof("Detected migration to %s", consensusType.Type) + if configMetadata == nil { return nil } @@ -1431,6 +1440,7 @@ func (c *Chain) getInFlightConfChange() *raftpb.ConfChange { // newMetadata extract config metadata from the configuration block func (c *Chain) newConfigMetadata(block *common.Block) (*etcdraft.ConfigMetadata, *orderer.ConsensusType) { + c.logger.Infof("Extract config metadata from the configuration block") metadata, consensusType, err := ConsensusMetadataFromConfigBlock(block) if err != nil { c.logger.Panicf("error reading consensus metadata: %s", err) @@ -1452,8 +1462,14 @@ func (c *Chain) ValidateConsensusMetadata(oldOrdererConfig, newOrdererConfig cha } if newOrdererConfig.ConsensusType() != "etcdraft" { - // This is a migration, so we don't know how to validate this config change. - return nil + if newOrdererConfig.ConsensusType() == "BFT" { + // This is a migration, so we don't know how to validate this config change. + return nil + } else { + c.logger.Panicf("illegal consensus type detected during consensus metadata validation: %s", newOrdererConfig.ConsensusType()) + panic("illegal consensus type detected during consensus metadata validation") + + } } if oldOrdererConfig == nil { diff --git a/orderer/consensus/etcdraft/chain_test.go b/orderer/consensus/etcdraft/chain_test.go index 88c82a80573..bc1ea8a4e16 100644 --- a/orderer/consensus/etcdraft/chain_test.go +++ b/orderer/consensus/etcdraft/chain_test.go @@ -678,6 +678,7 @@ var _ = Describe("Chain", func() { "ConsensusType": { Version: 1, Value: marshalOrPanic(&orderer.ConsensusType{ + Type: "etcdraft", Metadata: marshalOrPanic(consenterMetadata), }), }, @@ -711,6 +712,7 @@ var _ = Describe("Chain", func() { "ConsensusType": { Version: 1, Value: marshalOrPanic(&orderer.ConsensusType{ + Type: "etcdraft", Metadata: marshalOrPanic(metadata), }), }, @@ -1426,6 +1428,7 @@ var _ = Describe("Chain", func() { "ConsensusType": { Version: 1, Value: marshalOrPanic(&orderer.ConsensusType{ + Type: "etcdraft", Metadata: marshalOrPanic(metadata), }), }, @@ -1770,6 +1773,7 @@ var _ = Describe("Chain", func() { "ConsensusType": { Version: 1, Value: marshalOrPanic(&orderer.ConsensusType{ + Type: "etcdraft", Metadata: marshalOrPanic(metadata), }), }, @@ -1853,6 +1857,7 @@ var _ = Describe("Chain", func() { "ConsensusType": { Version: 1, Value: marshalOrPanic(&orderer.ConsensusType{ + Type: "etcdraft", Metadata: marshalOrPanic(metadata), }), }, @@ -1893,6 +1898,7 @@ var _ = Describe("Chain", func() { "ConsensusType": { Version: 1, Value: marshalOrPanic(&orderer.ConsensusType{ + Type: "etcdraft", Metadata: marshalOrPanic(metadata), }), }, @@ -1935,6 +1941,7 @@ var _ = Describe("Chain", func() { "ConsensusType": { Version: 1, Value: marshalOrPanic(&orderer.ConsensusType{ + Type: "etcdraft", Metadata: marshalOrPanic(metadata), }), }, diff --git a/orderer/consensus/etcdraft/util.go b/orderer/consensus/etcdraft/util.go index f734fc3c1b4..2c4a3b72082 100644 --- a/orderer/consensus/etcdraft/util.go +++ b/orderer/consensus/etcdraft/util.go @@ -9,6 +9,7 @@ package etcdraft import ( "crypto/x509" "encoding/pem" + "github.com/hyperledger/fabric-protos-go/orderer/smartbft" "time" "github.com/golang/protobuf/proto" @@ -90,6 +91,16 @@ func MetadataFromConfigValue(configValue *common.ConfigValue) (*etcdraft.ConfigM return nil, nil, errors.Wrap(err, "failed to unmarshal consensusType config update") } + if consensusTypeValue.Type != "etcdraft" { + if consensusTypeValue.Type == "BFT" { + updatedBftMetadata := &smartbft.Options{} + if err := proto.Unmarshal(consensusTypeValue.Metadata, updatedBftMetadata); err != nil { + return nil, nil, errors.Wrap(err, "failed to unmarshal updated (new) bft metadata configuration") + } + } + return nil, consensusTypeValue, nil + } + updatedMetadata := &etcdraft.ConfigMetadata{} if err := proto.Unmarshal(consensusTypeValue.Metadata, updatedMetadata); err != nil { return nil, nil, errors.Wrap(err, "failed to unmarshal updated (new) etcdraft metadata configuration") diff --git a/orderer/consensus/etcdraft/validator_test.go b/orderer/consensus/etcdraft/validator_test.go index 1a31f6bc536..c0a6817b9f7 100644 --- a/orderer/consensus/etcdraft/validator_test.go +++ b/orderer/consensus/etcdraft/validator_test.go @@ -113,6 +113,7 @@ var _ = Describe("Metadata Validation", func() { It("fails when old consensus metadata is not well-formed", func() { oldOrdererConf := mockOrderer([]byte("test")) newOrdererConf := mockOrderer([]byte("test")) + newOrdererConf.ConsensusTypeReturns("etcdraft") Expect(func() { chain.ValidateConsensusMetadata(oldOrdererConf, newOrdererConf, false) }).To(Panic()) @@ -122,6 +123,7 @@ var _ = Describe("Metadata Validation", func() { oldBytes, _ := proto.Marshal(&raftprotos.ConfigMetadata{}) oldOrdererConf := mockOrderer(oldBytes) newOrdererConf := mockOrderer([]byte("test")) + newOrdererConf.ConsensusTypeReturns("etcdraft") Expect(chain.ValidateConsensusMetadata(oldOrdererConf, newOrdererConf, false)).NotTo(Succeed()) }) }) @@ -177,6 +179,7 @@ var _ = Describe("Metadata Validation", func() { newOrdererConfig.OrganizationsReturns(map[string]channelconfig.OrdererOrg{ "org1": org1, }) + newOrdererConfig.ConsensusTypeReturns("etcdraft") newChannel = false })