[FAB-5274] Make WriteBlock asynchronous

In general, each consensus plugin for the ordering service has a single
thread which is responsible for receiving ordered messages, and writing
them into blocks.

This is a good thing, because it makes the consensus implementations
simpler and less error prone.  However, for operations like WriteBlock
which require some computations to be performed, followed by IO puts a
serious synchronous block in the path of these consensus threads.  While
the block is being written, the thread cannot do other things like drain
the queue or build the next block to write.

This CR decouples the actual writing of the block from the consenting
thread.  It accomplishes this by acquiring a lock in WriteBlock, then
spawning a go routine to go do the IO work, which releases the lock once
done.

While the lock is held, the consenter thread may perform other work,
but, if it gets to the point where it must construct a new block, the
CreateNextBlock function also acquires the lock, ensuring that the
integrity of the hash chain is not broken.

In non-scientific local testing with Solo, this produced an
approximate 30% increase in overall orderer throughput.

Change-Id: I11e2f9a716d415726c48066abe88a11a6e2fd1e0
Signed-off-by: Jason Yellick <jyellick@us.ibm.com>

orderer/common/multichannel/blockwriter.go

@@ -0,0 +1,209 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package multichannel
+
+import (
+	"sync"
+
+	"github.com/hyperledger/fabric/common/configtx"
+	configtxapi "github.com/hyperledger/fabric/common/configtx/api"
+	"github.com/hyperledger/fabric/common/crypto"
+	"github.com/hyperledger/fabric/common/util"
+	"github.com/hyperledger/fabric/orderer/common/ledger"
+	cb "github.com/hyperledger/fabric/protos/common"
+	"github.com/hyperledger/fabric/protos/utils"
+
+	"github.com/golang/protobuf/proto"
+)
+
+type blockWriterSupport interface {
+	crypto.LocalSigner
+	ledger.ReadWriter
+	configtxapi.Manager
+}
+
+// BlockWriter efficiently writes the blockchain to disk.
+// To safely use BlockWriter, only one thread should interact with it.
+// BlockWriter will spawn additional committing go routines and handle locking
+// so that these other go routines safely interact with the calling one.
+type BlockWriter struct {
+	support            blockWriterSupport
+	registrar          *Registrar
+	lastConfigBlockNum uint64
+	lastConfigSeq      uint64
+	lastBlock          *cb.Block
+	committingBlock    sync.Mutex
+}
+
+func newBlockWriter(lastBlock *cb.Block, r *Registrar, support blockWriterSupport) *BlockWriter {
+	bw := &BlockWriter{
+		support:       support,
+		lastConfigSeq: support.Sequence(),
+		lastBlock:     lastBlock,
+		registrar:     r,
+	}
+
+	// If this is the genesis block, the lastconfig field may be empty, and, the last config block is necessarily block 0
+	// so no need to initialize lastConfig
+	if lastBlock.Header.Number != 0 {
+		var err error
+		bw.lastConfigBlockNum, err = utils.GetLastConfigIndexFromBlock(lastBlock)
+		if err != nil {
+			logger.Panicf("[channel: %s] Error extracting last config block from block metadata: %s", support.ChainID(), err)
+		}
+	}
+
+	logger.Debugf("[channel: %s] Creating block writer for tip of chain (blockNumber=%d, lastConfigBlockNum=%d, lastConfigSeq=%d)", support.ChainID(), lastBlock.Header.Number, bw.lastConfigBlockNum, bw.lastConfigSeq)
+	return bw
+}
+
+// CreateNextBlock creates a new block with the next block number, and the given contents.
+func (bw *BlockWriter) CreateNextBlock(messages []*cb.Envelope) *cb.Block {
+	previousBlockHash := bw.lastBlock.Header.Hash()
+
+	data := &cb.BlockData{
+		Data: make([][]byte, len(messages)),
+	}
+
+	var err error
+	for i, msg := range messages {
+		data.Data[i], err = proto.Marshal(msg)
+		if err != nil {
+			logger.Panicf("Could not marshal envelope: %s", err)
+		}
+	}
+
+	block := cb.NewBlock(bw.lastBlock.Header.Number+1, previousBlockHash)
+	block.Header.DataHash = data.Hash()
+	block.Data = data
+
+	return block
+}
+
+// WriteConfigBlock should be invoked for blocks which contain a config transaction.
+// This call will block until the new config has taken effect, then will return
+// while the block is written asynchronously to disk.
+func (bw *BlockWriter) WriteConfigBlock(block *cb.Block, encodedMetadataValue []byte) {
+	ctx, err := utils.ExtractEnvelope(block, 0)
+	if err != nil {
+		logger.Panicf("Told to write a config block, but could not get configtx: %s", err)
+	}
+
+	payload, err := utils.UnmarshalPayload(ctx.Payload)
+	if err != nil {
+		logger.Panicf("Told to write a config block, but configtx payload is invalid: %s", err)
+	}
+
+	if payload.Header == nil {
+		logger.Panicf("Told to write a config block, but configtx payload header is missing")
+	}
+
+	chdr, err := utils.UnmarshalChannelHeader(payload.Header.ChannelHeader)
+	if err != nil {
+		logger.Panicf("Told to write a config block with an invalid channel header: %s", err)
+	}
+
+	switch chdr.Type {
+	case int32(cb.HeaderType_ORDERER_TRANSACTION):
+		newChannelConfig, err := utils.UnmarshalEnvelope(payload.Data)
+		if err != nil {
+			logger.Panicf("Told to write a config block with new channel, but did not have config update embedded: %s", err)
+		}
+		bw.registrar.newChain(newChannelConfig)
+	case int32(cb.HeaderType_CONFIG):
+		configEnvelope, err := configtx.UnmarshalConfigEnvelope(payload.Data)
+		if err != nil {
+			logger.Panicf("Told to write a config block with new channel, but did not have config envelope encoded: %s", err)
+		}
+
+		err = bw.support.Apply(configEnvelope)
+		if err != nil {
+			logger.Panicf("Told to write a config block with new config, but could not apply it: %s", err)
+		}
+	default:
+		logger.Panicf("Told to write a config block with unknown header type: %v", chdr.Type)
+	}
+
+	bw.WriteBlock(block, encodedMetadataValue)
+}
+
+// WriteBlock should be invoked for blocks which contain normal transactions.
+// It sets the target block as the pending next block, and returns before it is committed.
+// Before returning, it acquires the committing lock, and spawns a go routine which will
+// annotate the block with metadata and signatures, and write the block to the ledger
+// then release the lock.  This allows the calling thread to begin assembling the next block
+// before the commit phase is complete.
+func (bw *BlockWriter) WriteBlock(block *cb.Block, encodedMetadataValue []byte) {
+	bw.committingBlock.Lock()
+	bw.lastBlock = block
+
+	go func() {
+		defer bw.committingBlock.Unlock()
+		bw.commitBlock(encodedMetadataValue)
+	}()
+}
+
+// commitBlock should only ever be invoked with the bw.committingBlock held
+// this ensures that the encoded config sequence numbers stay in sync
+func (bw *BlockWriter) commitBlock(encodedMetadataValue []byte) {
+	// Set the orderer-related metadata field
+	if encodedMetadataValue != nil {
+		bw.lastBlock.Metadata.Metadata[cb.BlockMetadataIndex_ORDERER] = utils.MarshalOrPanic(&cb.Metadata{Value: encodedMetadataValue})
+	}
+	bw.addBlockSignature(bw.lastBlock)
+	bw.addLastConfigSignature(bw.lastBlock)
+
+	err := bw.support.Append(bw.lastBlock)
+	if err != nil {
+		logger.Panicf("[channel: %s] Could not append block: %s", bw.support.ChainID(), err)
+	}
+	logger.Debugf("[channel: %s] Wrote block %d", bw.support.ChainID(), bw.lastBlock.GetHeader().Number)
+}
+
+func (bw *BlockWriter) addBlockSignature(block *cb.Block) {
+	blockSignature := &cb.MetadataSignature{
+		SignatureHeader: utils.MarshalOrPanic(utils.NewSignatureHeaderOrPanic(bw.support)),
+	}
+
+	// Note, this value is intentionally nil, as this metadata is only about the signature, there is no additional metadata
+	// information required beyond the fact that the metadata item is signed.
+	blockSignatureValue := []byte(nil)
+
+	blockSignature.Signature = utils.SignOrPanic(bw.support, util.ConcatenateBytes(blockSignatureValue, blockSignature.SignatureHeader, block.Header.Bytes()))
+
+	block.Metadata.Metadata[cb.BlockMetadataIndex_SIGNATURES] = utils.MarshalOrPanic(&cb.Metadata{
+		Value: blockSignatureValue,
+		Signatures: []*cb.MetadataSignature{
+			blockSignature,
+		},
+	})
+}
+
+func (bw *BlockWriter) addLastConfigSignature(block *cb.Block) {
+	configSeq := bw.support.Sequence()
+	if configSeq > bw.lastConfigSeq {
+		logger.Debugf("[channel: %s] Detected lastConfigSeq transitioning from %d to %d, setting lastConfigBlockNum from %d to %d", bw.support.ChainID(), bw.lastConfigSeq, configSeq, bw.lastConfigBlockNum, block.Header.Number)
+		bw.lastConfigBlockNum = block.Header.Number
+		bw.lastConfigSeq = configSeq
+	}
+
+	lastConfigSignature := &cb.MetadataSignature{
+		SignatureHeader: utils.MarshalOrPanic(utils.NewSignatureHeaderOrPanic(bw.support)),
+	}
+
+	lastConfigValue := utils.MarshalOrPanic(&cb.LastConfig{Index: bw.lastConfigBlockNum})
+	logger.Debugf("[channel: %s] About to write block, setting its LAST_CONFIG to %d", bw.support.ChainID(), bw.lastConfigBlockNum)
+
+	lastConfigSignature.Signature = utils.SignOrPanic(bw.support, util.ConcatenateBytes(lastConfigValue, lastConfigSignature.SignatureHeader, block.Header.Bytes()))
+
+	block.Metadata.Metadata[cb.BlockMetadataIndex_LAST_CONFIG] = utils.MarshalOrPanic(&cb.Metadata{
+		Value: lastConfigValue,
+		Signatures: []*cb.MetadataSignature{
+			lastConfigSignature,
+		},
+	})
+}

orderer/common/multichannel/blockwriter_test.go

@@ -0,0 +1,188 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package multichannel
+
+import (
+	"testing"
+
+	// "github.com/golang/protobuf/proto"
+	"github.com/hyperledger/fabric/common/configtx/tool/provisional"
+	"github.com/hyperledger/fabric/common/crypto"
+	mockconfigtx "github.com/hyperledger/fabric/common/mocks/configtx"
+	"github.com/hyperledger/fabric/orderer/common/ledger"
+	cb "github.com/hyperledger/fabric/protos/common"
+	"github.com/hyperledger/fabric/protos/utils"
+	"github.com/stretchr/testify/assert"
+)
+
+type mockBlockWriterSupport struct {
+	*mockconfigtx.Manager
+	crypto.LocalSigner
+	ledger.ReadWriter
+}
+
+func TestCreateBlock(t *testing.T) {
+	seedBlock := cb.NewBlock(7, []byte("lasthash"))
+	seedBlock.Data.Data = [][]byte{[]byte("somebytes")}
+
+	bw := &BlockWriter{lastBlock: seedBlock}
+	block := bw.CreateNextBlock([]*cb.Envelope{
+		&cb.Envelope{Payload: []byte("some other bytes")},
+	})
+
+	assert.Equal(t, seedBlock.Header.Number+1, block.Header.Number)
+	assert.Equal(t, block.Data.Hash(), block.Header.DataHash)
+	assert.Equal(t, seedBlock.Header.Hash(), block.Header.PreviousHash)
+}
+
+func TestBlockSignature(t *testing.T) {
+	bw := &BlockWriter{
+		support: &mockBlockWriterSupport{
+			LocalSigner: mockCrypto(),
+		},
+	}
+
+	block := cb.NewBlock(7, []byte("foo"))
+	bw.addBlockSignature(block)
+
+	md := utils.GetMetadataFromBlockOrPanic(block, cb.BlockMetadataIndex_SIGNATURES)
+	assert.Nil(t, md.Value, "Value is empty in this case")
+	assert.NotNil(t, md.Signatures, "Should have signature")
+}
+
+func TestBlockLastConfig(t *testing.T) {
+	lastConfigSeq := uint64(6)
+	newConfigSeq := lastConfigSeq + 1
+	newBlockNum := uint64(9)
+
+	bw := &BlockWriter{
+		support: &mockBlockWriterSupport{
+			LocalSigner: mockCrypto(),
+			Manager: &mockconfigtx.Manager{
+				SequenceVal: newConfigSeq,
+			},
+		},
+		lastConfigSeq: lastConfigSeq,
+	}
+
+	block := cb.NewBlock(newBlockNum, []byte("foo"))
+	bw.addLastConfigSignature(block)
+
+	assert.Equal(t, newBlockNum, bw.lastConfigBlockNum)
+	assert.Equal(t, newConfigSeq, bw.lastConfigSeq)
+
+	md := utils.GetMetadataFromBlockOrPanic(block, cb.BlockMetadataIndex_LAST_CONFIG)
+	assert.NotNil(t, md.Value, "Value not be empty in this case")
+	assert.NotNil(t, md.Signatures, "Should have signature")
+
+	lc := utils.GetLastConfigIndexFromBlockOrPanic(block)
+	assert.Equal(t, newBlockNum, lc)
+}
+
+func TestWriteConfigBlock(t *testing.T) {
+	// TODO, use assert.PanicsWithValue once available
+	t.Run("EmptyBlock", func(t *testing.T) {
+		assert.Panics(t, func() { (&BlockWriter{}).WriteConfigBlock(&cb.Block{}, nil) })
+	})
+	t.Run("BadPayload", func(t *testing.T) {
+		assert.Panics(t, func() {
+			(&BlockWriter{}).WriteConfigBlock(&cb.Block{
+				Data: &cb.BlockData{
+					Data: [][]byte{
+						utils.MarshalOrPanic(&cb.Envelope{Payload: []byte("bad")}),
+					},
+				},
+			}, nil)
+		})
+	})
+	t.Run("MissingHeader", func(t *testing.T) {
+		assert.Panics(t, func() {
+			(&BlockWriter{}).WriteConfigBlock(&cb.Block{
+				Data: &cb.BlockData{
+					Data: [][]byte{
+						utils.MarshalOrPanic(&cb.Envelope{
+							Payload: utils.MarshalOrPanic(&cb.Payload{}),
+						}),
+					},
+				},
+			}, nil)
+		})
+	})
+	t.Run("BadChannelHeader", func(t *testing.T) {
+		assert.Panics(t, func() {
+			(&BlockWriter{}).WriteConfigBlock(&cb.Block{
+				Data: &cb.BlockData{
+					Data: [][]byte{
+						utils.MarshalOrPanic(&cb.Envelope{
+							Payload: utils.MarshalOrPanic(&cb.Payload{
+								Header: &cb.Header{
+									ChannelHeader: []byte("bad"),
+								},
+							}),
+						}),
+					},
+				},
+			}, nil)
+		})
+	})
+	t.Run("BadChannelHeaderType", func(t *testing.T) {
+		assert.Panics(t, func() {
+			(&BlockWriter{}).WriteConfigBlock(&cb.Block{
+				Data: &cb.BlockData{
+					Data: [][]byte{
+						utils.MarshalOrPanic(&cb.Envelope{
+							Payload: utils.MarshalOrPanic(&cb.Payload{
+								Header: &cb.Header{
+									ChannelHeader: utils.MarshalOrPanic(&cb.ChannelHeader{}),
+								},
+							}),
+						}),
+					},
+				},
+			}, nil)
+		})
+	})
+}
+
+func TestGoodWriteConfig(t *testing.T) {
+	l := NewRAMLedger(10)
+
+	bw := &BlockWriter{
+		support: &mockBlockWriterSupport{
+			LocalSigner: mockCrypto(),
+			ReadWriter:  l,
+			Manager:     &mockconfigtx.Manager{},
+		},
+	}
+
+	ctx := makeConfigTx(provisional.TestChainID, 1)
+	block := cb.NewBlock(1, genesisBlock.Header.Hash())
+	block.Data.Data = [][]byte{utils.MarshalOrPanic(ctx)}
+	consenterMetadata := []byte("foo")
+	bw.WriteConfigBlock(block, consenterMetadata)
+
+	// Wait for the commit to complete
+	bw.committingBlock.Lock()
+	bw.committingBlock.Unlock()
+
+	cBlock := ledger.GetBlock(l, block.Header.Number)
+	assert.Equal(t, block.Header, cBlock.Header)
+	assert.Equal(t, block.Data, cBlock.Data)
+
+	omd := utils.GetMetadataFromBlockOrPanic(block, cb.BlockMetadataIndex_ORDERER)
+	assert.Equal(t, consenterMetadata, omd.Value)
+}