diff --git a/orderer/common/bootstrap/provisional/provisional.go b/orderer/common/bootstrap/provisional/provisional.go index fd84135b9ca..d80e31e07d0 100644 --- a/orderer/common/bootstrap/provisional/provisional.go +++ b/orderer/common/bootstrap/provisional/provisional.go @@ -84,7 +84,8 @@ func New(conf *config.TopLevel) Generator { chainID: TestChainID, consensusType: conf.Genesis.OrdererType, batchSize: &ab.BatchSize{ - MaxMessageCount: conf.Genesis.BatchSize.MaxMessageCount, + MaxMessageCount: conf.Genesis.BatchSize.MaxMessageCount, + AbsoluteMaxBytes: conf.Genesis.BatchSize.AbsoluteMaxBytes, }, batchTimeout: conf.Genesis.BatchTimeout.String(), } diff --git a/orderer/common/sharedconfig/sharedconfig.go b/orderer/common/sharedconfig/sharedconfig.go index a312aaaa403..3e860420e7c 100644 --- a/orderer/common/sharedconfig/sharedconfig.go +++ b/orderer/common/sharedconfig/sharedconfig.go @@ -194,8 +194,11 @@ func (pm *ManagerImpl) ProposeConfig(configItem *cb.ConfigurationItem) error { if err := proto.Unmarshal(configItem.Value, batchSize); err != nil { return fmt.Errorf("Unmarshaling error for BatchSize: %s", err) } - if batchSize.MaxMessageCount <= 0 { - return fmt.Errorf("Attempted to set the batch size max message count to %d which is less than or equal to 0", batchSize.MaxMessageCount) + if batchSize.MaxMessageCount == 0 { + return fmt.Errorf("Attempted to set the batch size max message count to an invalid value: 0") + } + if batchSize.AbsoluteMaxBytes == 0 { + return fmt.Errorf("Attempted to set the batch size absolute max bytes to an invalid value: 0") } pm.pendingConfig.batchSize = batchSize case BatchTimeoutKey: diff --git a/orderer/common/sharedconfig/sharedconfig_test.go b/orderer/common/sharedconfig/sharedconfig_test.go index 2eba43a1843..899cf3eb544 100644 --- a/orderer/common/sharedconfig/sharedconfig_test.go +++ b/orderer/common/sharedconfig/sharedconfig_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + cb "github.com/hyperledger/fabric/protos/common" ab "github.com/hyperledger/fabric/protos/orderer" "github.com/hyperledger/fabric/protos/utils" @@ -132,51 +134,63 @@ func TestConsensusType(t *testing.T) { } func TestBatchSize(t *testing.T) { - endBatchSize := struct{ MaxMessageCount uint32 }{ - MaxMessageCount: uint32(10), - } - invalidMessage := &cb.ConfigurationItem{ - Type: cb.ConfigurationItem_Orderer, - Key: BatchSizeKey, - Value: []byte("Garbage Data"), - } - zeroBatchSize := &cb.ConfigurationItem{ - Type: cb.ConfigurationItem_Orderer, - Key: BatchSizeKey, - Value: utils.MarshalOrPanic(&ab.BatchSize{MaxMessageCount: 0}), - } - validMessage := &cb.ConfigurationItem{ - Type: cb.ConfigurationItem_Orderer, - Key: BatchSizeKey, - Value: utils.MarshalOrPanic(&ab.BatchSize{MaxMessageCount: endBatchSize.MaxMessageCount}), - } - m := NewManagerImpl() - m.BeginConfig() - err := m.ProposeConfig(validMessage) - if err != nil { - t.Fatalf("Error applying valid config: %s", err) - } + validMaxMessageCount := uint32(10) + validAbsoluteMaxBytes := uint32(1000) - err = m.ProposeConfig(invalidMessage) - if err == nil { - t.Fatalf("Should have failed on invalid message") - } - - err = m.ProposeConfig(zeroBatchSize) - if err == nil { - t.Fatalf("Should have rejected batch size of 0") - } + t.Run("ValidConfiguration", func(t *testing.T) { + m := NewManagerImpl() + m.BeginConfig() + err := m.ProposeConfig(&cb.ConfigurationItem{ + Type: cb.ConfigurationItem_Orderer, + Key: BatchSizeKey, + Value: utils.MarshalOrPanic(&ab.BatchSize{MaxMessageCount: validMaxMessageCount, AbsoluteMaxBytes: validAbsoluteMaxBytes}), + }) + assert.Nil(t, err, "Error applying valid config: %s", err) + m.CommitConfig() + if m.BatchSize().MaxMessageCount != validMaxMessageCount { + t.Fatalf("Got batch size max message count of %d. Expected: %d", m.BatchSize().MaxMessageCount, validMaxMessageCount) + } + if m.BatchSize().AbsoluteMaxBytes != validAbsoluteMaxBytes { + t.Fatalf("Got batch size absolute max bytes of %d. Expected: %d", m.BatchSize().AbsoluteMaxBytes, validAbsoluteMaxBytes) + } + }) - m.CommitConfig() + t.Run("UnserializableConfiguration", func(t *testing.T) { + m := NewManagerImpl() + m.BeginConfig() + err := m.ProposeConfig(&cb.ConfigurationItem{ + Type: cb.ConfigurationItem_Orderer, + Key: BatchSizeKey, + Value: []byte("Garbage Data"), + }) + assert.NotNil(t, err, "Should have failed on invalid message") + m.CommitConfig() + }) - nowBatchSize := struct{ MaxMessageCount uint32 }{ - MaxMessageCount: m.BatchSize().MaxMessageCount, - } + t.Run("ZeroMaxMessageCount", func(t *testing.T) { + m := NewManagerImpl() + m.BeginConfig() + err := m.ProposeConfig(&cb.ConfigurationItem{ + Type: cb.ConfigurationItem_Orderer, + Key: BatchSizeKey, + Value: utils.MarshalOrPanic(&ab.BatchSize{MaxMessageCount: 0, AbsoluteMaxBytes: validAbsoluteMaxBytes}), + }) + assert.NotNil(t, err, "Should have rejected batch size max message count of 0") + m.CommitConfig() + }) - if nowBatchSize.MaxMessageCount != endBatchSize.MaxMessageCount { - t.Fatalf("Got batch size max message count of %d. Expected: %d", nowBatchSize.MaxMessageCount, endBatchSize.MaxMessageCount) - } + t.Run("ZeroAbsoluteMaxBytes", func(t *testing.T) { + m := NewManagerImpl() + m.BeginConfig() + err := m.ProposeConfig(&cb.ConfigurationItem{ + Type: cb.ConfigurationItem_Orderer, + Key: BatchSizeKey, + Value: utils.MarshalOrPanic(&ab.BatchSize{MaxMessageCount: validMaxMessageCount, AbsoluteMaxBytes: 0}), + }) + assert.NotNil(t, err, "Should have rejected batch size absolute max message bytes of 0") + m.CommitConfig() + }) } func TestBatchTimeout(t *testing.T) { diff --git a/orderer/kafka/config_test.go b/orderer/kafka/config_test.go index 5c25148f444..a55eae15e5d 100644 --- a/orderer/kafka/config_test.go +++ b/orderer/kafka/config_test.go @@ -59,7 +59,8 @@ var testConf = &config.TopLevel{ OrdererType: "kafka", BatchTimeout: 500 * time.Millisecond, BatchSize: config.BatchSize{ - MaxMessageCount: 100, + MaxMessageCount: 100, + AbsoluteMaxBytes: 10 * 1024 * 1024, }, }, } diff --git a/orderer/localconfig/config.go b/orderer/localconfig/config.go index d6f829f9909..eacc608bb24 100644 --- a/orderer/localconfig/config.go +++ b/orderer/localconfig/config.go @@ -59,7 +59,8 @@ type Genesis struct { // BatchSize contains configuration affecting the size of batches type BatchSize struct { - MaxMessageCount uint32 + MaxMessageCount uint32 + AbsoluteMaxBytes uint32 } // Profile contains configuration for Go pprof profiling @@ -141,7 +142,8 @@ var defaults = TopLevel{ OrdererType: "solo", BatchTimeout: 10 * time.Second, BatchSize: BatchSize{ - MaxMessageCount: 10, + MaxMessageCount: 10, + AbsoluteMaxBytes: 100000000, }, }, } @@ -197,6 +199,9 @@ func (c *TopLevel) completeInitialization() { case c.Genesis.BatchSize.MaxMessageCount == 0: logger.Infof("Genesis.BatchSize.MaxMessageCount unset, setting to %s", defaults.Genesis.BatchSize.MaxMessageCount) c.Genesis.BatchSize.MaxMessageCount = defaults.Genesis.BatchSize.MaxMessageCount + case c.Genesis.BatchSize.AbsoluteMaxBytes == 0: + logger.Infof("Genesis.BatchSize.AbsoluteMaxBytes unset, setting to %s", defaults.Genesis.BatchSize.AbsoluteMaxBytes) + c.Genesis.BatchSize.AbsoluteMaxBytes = defaults.Genesis.BatchSize.AbsoluteMaxBytes default: // A bit hacky, but its type makes it impossible to test for a nil value. // This may be overwritten by the Kafka orderer upon instantiation. diff --git a/orderer/orderer.yaml b/orderer/orderer.yaml index 3434f4167d6..f557d7c6c91 100644 --- a/orderer/orderer.yaml +++ b/orderer/orderer.yaml @@ -122,3 +122,6 @@ Genesis: # Max Message Count: The maximum number of messages to permit in a batch MaxMessageCount: 10 + # Absolute Max Bytes: The absolute maximum number of bytes allowed for + # the serialized messages in a batch. + AbsoluteMaxBytes: 100000000 diff --git a/protos/orderer/configuration.pb.go b/protos/orderer/configuration.pb.go index a73a67b9f2a..cf2a9d4f570 100644 --- a/protos/orderer/configuration.pb.go +++ b/protos/orderer/configuration.pb.go @@ -27,6 +27,9 @@ type BatchSize struct { // Simply specified as number of messages for now, in the future // we may want to allow this to be specified by size in bytes MaxMessageCount uint32 `protobuf:"varint,1,opt,name=maxMessageCount" json:"maxMessageCount,omitempty"` + // The byte count of the serialized messages in a batch cannot + // exceed this value. + AbsoluteMaxBytes uint32 `protobuf:"varint,2,opt,name=absoluteMaxBytes" json:"absoluteMaxBytes,omitempty"` } func (m *BatchSize) Reset() { *m = BatchSize{} } @@ -123,25 +126,26 @@ func init() { func init() { proto.RegisterFile("orderer/configuration.proto", fileDescriptor1) } var fileDescriptor1 = []byte{ - // 308 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x91, 0x41, 0x4b, 0xc3, 0x40, - 0x10, 0x85, 0x89, 0x96, 0xd6, 0x0e, 0xad, 0xc2, 0x0a, 0x12, 0xea, 0xa5, 0xc4, 0x4b, 0x40, 0x69, - 0x0e, 0xe2, 0x5d, 0x1a, 0x3c, 0x88, 0x08, 0x12, 0x7b, 0xf2, 0xb6, 0x49, 0xa7, 0xc9, 0xd2, 0x66, - 0x27, 0xcc, 0x6e, 0xc0, 0xf8, 0xeb, 0x25, 0x9b, 0x6d, 0x0f, 0x5e, 0x3c, 0xe5, 0x7d, 0x33, 0x2f, - 0xb3, 0xf3, 0x18, 0xb8, 0x25, 0xde, 0x22, 0x23, 0x27, 0x05, 0xe9, 0x9d, 0x2a, 0x5b, 0x96, 0x56, - 0x91, 0x5e, 0x35, 0x4c, 0x96, 0xc4, 0xc4, 0x37, 0x17, 0xd7, 0x05, 0xd5, 0x35, 0xe9, 0x64, 0xf8, - 0x0c, 0xdd, 0xe8, 0x0e, 0xe6, 0x29, 0x69, 0x83, 0xda, 0xb4, 0x66, 0xd3, 0x35, 0x28, 0x04, 0x8c, - 0x6c, 0xd7, 0x60, 0x18, 0x2c, 0x83, 0x78, 0x9a, 0x39, 0x1d, 0x3d, 0xc1, 0x74, 0x2d, 0x6d, 0x51, - 0x7d, 0xaa, 0x1f, 0x14, 0x31, 0x5c, 0xd5, 0xf2, 0xfb, 0x1d, 0x8d, 0x91, 0x25, 0xa6, 0xd4, 0x6a, - 0xeb, 0xbc, 0xf3, 0xec, 0x6f, 0x39, 0x8a, 0x61, 0xe6, 0x7e, 0xdb, 0xa8, 0x1a, 0xa9, 0xb5, 0x22, - 0x84, 0x89, 0x1d, 0xa4, 0x9f, 0x7e, 0xc4, 0xe8, 0x19, 0x2e, 0x53, 0x46, 0xb7, 0xf5, 0x07, 0x1d, - 0x54, 0xd1, 0x89, 0x1b, 0x18, 0x37, 0x4e, 0x79, 0xab, 0xa7, 0xbe, 0xbe, 0x55, 0x25, 0x1a, 0x1b, - 0x9e, 0x2d, 0x83, 0x78, 0x96, 0x79, 0xea, 0x73, 0xbc, 0xea, 0x92, 0xd1, 0x18, 0x3f, 0x40, 0xc0, - 0x48, 0xcb, 0xfa, 0x94, 0xa3, 0xd7, 0x51, 0x04, 0xb3, 0x97, 0xff, 0x3c, 0xf7, 0x30, 0x4f, 0x2b, - 0xa9, 0xb4, 0xdb, 0x87, 0xd8, 0x88, 0x05, 0x5c, 0xb8, 0xb7, 0x15, 0x9a, 0x30, 0x58, 0x9e, 0xc7, - 0xd3, 0xec, 0xc4, 0x7d, 0xc2, 0x37, 0xb9, 0xdb, 0xcb, 0x35, 0xd3, 0x1e, 0xd9, 0xf4, 0x09, 0xf3, - 0x41, 0x7a, 0xeb, 0x11, 0xd7, 0xab, 0xaf, 0x87, 0x52, 0xd9, 0xaa, 0xcd, 0x57, 0x05, 0xd5, 0x49, - 0xd5, 0x35, 0xc8, 0x07, 0xdc, 0x96, 0xc8, 0xc9, 0x4e, 0xe6, 0xac, 0x8a, 0xc4, 0x9d, 0xc3, 0x24, - 0xfe, 0x58, 0xf9, 0xd8, 0xf1, 0xe3, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd9, 0x31, 0x29, 0x3f, - 0xdb, 0x01, 0x00, 0x00, + // 328 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x91, 0x41, 0x6b, 0xe3, 0x30, + 0x10, 0x85, 0xc9, 0x6e, 0x48, 0x36, 0x43, 0xb2, 0xbb, 0x68, 0x61, 0x31, 0xe9, 0x25, 0xb8, 0x17, + 0xd3, 0x96, 0xf8, 0xd0, 0x3f, 0x50, 0x6c, 0x7a, 0x28, 0x25, 0x50, 0xdc, 0x9c, 0x7a, 0x93, 0x9d, + 0x89, 0x2d, 0x12, 0x6b, 0x8c, 0x46, 0x86, 0xb8, 0xbf, 0xbe, 0x58, 0x56, 0x72, 0x68, 0x0f, 0x3d, + 0xe9, 0x7d, 0x33, 0x0f, 0xcd, 0x3c, 0x06, 0xae, 0xc8, 0xec, 0xd0, 0xa0, 0x89, 0x0b, 0xd2, 0x7b, + 0x55, 0xb6, 0x46, 0x5a, 0x45, 0x7a, 0xdd, 0x18, 0xb2, 0x24, 0xa6, 0xbe, 0xb9, 0xfc, 0x57, 0x50, + 0x5d, 0x93, 0x8e, 0x87, 0x67, 0xe8, 0x86, 0xd7, 0xb0, 0x48, 0x49, 0x33, 0x6a, 0x6e, 0x79, 0xdb, + 0x35, 0x28, 0x04, 0x8c, 0x6d, 0xd7, 0x60, 0x30, 0x5a, 0x8d, 0xa2, 0x59, 0xe6, 0x74, 0x28, 0x61, + 0x96, 0x48, 0x5b, 0x54, 0xaf, 0xea, 0x1d, 0x45, 0x04, 0x7f, 0x6a, 0x79, 0xda, 0x20, 0xb3, 0x2c, + 0x31, 0xa5, 0x56, 0x5b, 0xe7, 0x5d, 0x64, 0x9f, 0xcb, 0xe2, 0x06, 0xfe, 0xca, 0x9c, 0xe9, 0xd8, + 0x5a, 0xdc, 0xc8, 0x53, 0xd2, 0x59, 0xe4, 0xe0, 0x87, 0xb3, 0x7e, 0xa9, 0x87, 0x11, 0xcc, 0xdd, + 0x88, 0xad, 0xaa, 0x91, 0x5a, 0x2b, 0x02, 0x98, 0xda, 0x41, 0xfa, 0x4d, 0xce, 0x18, 0x3e, 0xc0, + 0xef, 0xd4, 0xa0, 0x4b, 0xf8, 0x42, 0x47, 0x55, 0x74, 0xe2, 0x3f, 0x4c, 0x1a, 0xa7, 0xbc, 0xd5, + 0x53, 0x5f, 0xdf, 0xa9, 0x12, 0xd9, 0xba, 0xa9, 0xf3, 0xcc, 0x53, 0x9f, 0xf9, 0x49, 0x97, 0x06, + 0x99, 0xfd, 0x07, 0x02, 0xc6, 0x5a, 0xd6, 0x97, 0xcc, 0xbd, 0x0e, 0x43, 0x98, 0x3f, 0x7e, 0xe7, + 0xb9, 0x85, 0x45, 0x5a, 0x49, 0xa5, 0xdd, 0x3e, 0x64, 0x58, 0x2c, 0xe1, 0x97, 0x9b, 0xad, 0x90, + 0x83, 0xd1, 0xea, 0x67, 0x34, 0xcb, 0x2e, 0xdc, 0x27, 0x7c, 0x96, 0xfb, 0x83, 0x4c, 0x0c, 0x1d, + 0xd0, 0x70, 0x9f, 0x30, 0x1f, 0xa4, 0xb7, 0x9e, 0x31, 0x59, 0xbf, 0xdd, 0x95, 0xca, 0x56, 0x6d, + 0xbe, 0x2e, 0xa8, 0x8e, 0xab, 0xae, 0x41, 0x73, 0xc4, 0x5d, 0x89, 0x26, 0xde, 0xcb, 0xdc, 0xa8, + 0x22, 0x76, 0xa7, 0xe3, 0xd8, 0x1f, 0x36, 0x9f, 0x38, 0xbe, 0xff, 0x08, 0x00, 0x00, 0xff, 0xff, + 0x6e, 0xaf, 0xb7, 0x25, 0x07, 0x02, 0x00, 0x00, } diff --git a/protos/orderer/configuration.proto b/protos/orderer/configuration.proto index 493155bf1fa..89c2014cd52 100644 --- a/protos/orderer/configuration.proto +++ b/protos/orderer/configuration.proto @@ -37,6 +37,9 @@ message BatchSize { // Simply specified as number of messages for now, in the future // we may want to allow this to be specified by size in bytes uint32 maxMessageCount = 1; + // The byte count of the serialized messages in a batch cannot + // exceed this value. + uint32 absoluteMaxBytes = 2; } message BatchTimeout { diff --git a/protos/utils/blockutils.go b/protos/utils/blockutils.go index f848a5b2c4d..75dfa9a278c 100644 --- a/protos/utils/blockutils.go +++ b/protos/utils/blockutils.go @@ -84,7 +84,7 @@ func CopyBlockMetadata(src *cb.Block, dst *cb.Block) { InitBlockMetadata(dst) } -// CopyBlockMetadata copies metadata from one block into another +// InitBlockMetadata copies metadata from one block into another func InitBlockMetadata(block *cb.Block) { if block.Metadata == nil { block.Metadata = &cb.BlockMetadata{Metadata: [][]byte{[]byte{}, []byte{}, []byte{}}}