diff --git a/common/ledger/testutil/test_helper.go b/common/ledger/testutil/test_helper.go index 6ded5ae61bb..4e11c79faf8 100644 --- a/common/ledger/testutil/test_helper.go +++ b/common/ledger/testutil/test_helper.go @@ -21,15 +21,12 @@ import ( "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric/common/configtx/test" - "github.com/hyperledger/fabric/common/configtx/tool/provisional" "github.com/hyperledger/fabric/common/util" lutils "github.com/hyperledger/fabric/core/ledger/util" "github.com/hyperledger/fabric/protos/common" pb "github.com/hyperledger/fabric/protos/peer" ptestutils "github.com/hyperledger/fabric/protos/testutils" "github.com/hyperledger/fabric/protos/utils" - - genesisconfig "github.com/hyperledger/fabric/common/configtx/tool/localconfig" ) //BlockGenerator generates a series of blocks for testing @@ -147,9 +144,3 @@ func newBlock(env []*common.Envelope, blockNum uint64, previousHash []byte) *com return block } - -func MakeGenesisBlock() *common.Block { - genConf := genesisconfig.Load(genesisconfig.SampleInsecureProfile) - gb := provisional.New(genConf).GenesisBlock() - return gb -} diff --git a/test/tools/ledgerbenchmarks/chainmgmt/block_gen.go b/test/tools/ledgerbenchmarks/chainmgmt/block_gen.go new file mode 100644 index 00000000000..3deebde9d44 --- /dev/null +++ b/test/tools/ledgerbenchmarks/chainmgmt/block_gen.go @@ -0,0 +1,106 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chainmgmt + +import ( + "sync" + + "github.com/golang/protobuf/proto" + "github.com/hyperledger/fabric/protos/common" + benchcommon "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common" +) + +const ( + numConcurrentTxEnvCreators = 30 +) + +type txEnvBytes []byte + +// blkGenerator generates blocks in sequence. One instance of blkGenerator is maintained for each chain +type blkGenerator struct { + batchConf *BatchConf + blockNum uint64 + previousBlockHash []byte + + srQueue chan SimulationResult + txQueue chan txEnvBytes + wg *sync.WaitGroup +} + +func newBlkGenerator(batchConf *BatchConf, startingBlockNum uint64, previousBlockHash []byte) *blkGenerator { + bg := &blkGenerator{ + batchConf, + startingBlockNum, + previousBlockHash, + make(chan SimulationResult, batchConf.BatchSize), + make(chan txEnvBytes, batchConf.BatchSize), + &sync.WaitGroup{}, + } + bg.startTxEnvCreators() + return bg +} + +func (bg *blkGenerator) startTxEnvCreators() { + for i := 0; i < numConcurrentTxEnvCreators; i++ { + go bg.startTxEnvCreator() + } +} + +func (bg *blkGenerator) startTxEnvCreator() { + bg.wg.Add(1) + for sr := range bg.srQueue { + txEnv, err := createTxEnv(sr) + benchcommon.PanicOnError(err) + txEnvBytes, err := proto.Marshal(txEnv) + benchcommon.PanicOnError(err) + bg.txQueue <- txEnvBytes + } + bg.wg.Done() +} + +func (bg *blkGenerator) addTx(sr SimulationResult) { + bg.srQueue <- sr +} + +func (bg *blkGenerator) nextBlock() *common.Block { + block := common.NewBlock(bg.blockNum, bg.previousBlockHash) + numTx := 0 + for txEnvBytes := range bg.txQueue { + numTx++ + block.Data.Data = append(block.Data.Data, txEnvBytes) + if numTx == bg.batchConf.BatchSize { + break + } + } + // close() has been called and no pending tx + if len(block.Data.Data) == 0 { + return nil + } + block.Header.DataHash = block.Data.Hash() + block.Header.Number = bg.blockNum + block.Header.PreviousHash = bg.previousBlockHash + + bg.blockNum++ + bg.previousBlockHash = block.Header.Hash() + return block +} + +func (bg *blkGenerator) close() { + close(bg.srQueue) + bg.wg.Wait() + close(bg.txQueue) +} diff --git a/test/tools/ledgerbenchmarks/chainmgmt/chains.go b/test/tools/ledgerbenchmarks/chainmgmt/chains.go new file mode 100644 index 00000000000..f07d0c68500 --- /dev/null +++ b/test/tools/ledgerbenchmarks/chainmgmt/chains.go @@ -0,0 +1,142 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chainmgmt + +import ( + "fmt" + "sync" + + "github.com/hyperledger/fabric/common/configtx/test" + "github.com/hyperledger/fabric/core/ledger" + "github.com/hyperledger/fabric/core/ledger/ledgermgmt" + "github.com/hyperledger/fabric/protos/common" + benchcommon "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common" +) + +// ChainID is a type used for the ids for the chains for experiments +type ChainID int + +func (chainID ChainID) String() string { + return fmt.Sprintf("%s%04d", "chain_", chainID) +} + +// SimulationResult is a type used for simulation results +type SimulationResult []byte + +// chainsMgr manages chains for experiments +type chainsMgr struct { + mgrConf *ChainMgrConf + batchConf *BatchConf + initOp chainInitOp + chainsMap map[ChainID]*Chain + wg *sync.WaitGroup +} + +func newChainsMgr(mgrConf *ChainMgrConf, batchConf *BatchConf, initOp chainInitOp) *chainsMgr { + ledgermgmt.Initialize() + return &chainsMgr{mgrConf, batchConf, initOp, make(map[ChainID]*Chain), &sync.WaitGroup{}} +} + +func (m *chainsMgr) createOrOpenChains() []*Chain { + var ledgerInitFunc func(string) (ledger.PeerLedger, error) + switch m.initOp { + case ChainInitOpCreate: + ledgerInitFunc = createLedgerByID + case ChainInitOpOpen: + ledgerInitFunc = ledgermgmt.OpenLedger + default: + panic(fmt.Errorf("unknown chain init opeartion")) + } + + numChains := m.mgrConf.NumChains + for i := 0; i < numChains; i++ { + chainID := ChainID(i) + peerLedger, err := ledgerInitFunc(chainID.String()) + benchcommon.PanicOnError(err) + c := newChain(chainID, peerLedger, m) + m.chainsMap[chainID] = c + } + return m.chains() +} + +func (m *chainsMgr) chains() []*Chain { + chains := []*Chain{} + for _, chain := range m.chainsMap { + chains = append(chains, chain) + } + return chains +} + +func (m *chainsMgr) waitForChainsToExhaustAllBlocks() { + m.wg.Wait() + ledgermgmt.Close() +} + +// Chain extends ledger.PeerLedger and the experiments invoke ledger functions via a chain type +type Chain struct { + ledger.PeerLedger + ID ChainID + blkGenerator *blkGenerator + m *chainsMgr +} + +func newChain(id ChainID, peerLedger ledger.PeerLedger, m *chainsMgr) *Chain { + bcInfo, err := peerLedger.GetBlockchainInfo() + benchcommon.PanicOnError(err) + return &Chain{peerLedger, id, newBlkGenerator(m.batchConf, bcInfo.Height, bcInfo.CurrentBlockHash), m} +} + +func (c *Chain) startBlockPollingAndCommit() { + c.m.wg.Add(1) + go func() { + defer c.close() + for { + block := c.blkGenerator.nextBlock() + if block == nil { + break + } + benchcommon.PanicOnError(c.PeerLedger.Commit(block)) + } + }() +} + +// SubmitTx is expected to be called by an experiment for submitting the transactions +func (c *Chain) SubmitTx(sr SimulationResult) { + c.blkGenerator.addTx(sr) +} + +// Done is expected to be called by an experiment when the experiment does not have any more transactions to submit +func (c *Chain) Done() { + c.blkGenerator.close() +} + +// Commit overrides the Commit function in ledger.PeerLedger because, +// experiments are not expected to call Commit directly to the ledger +func (c *Chain) Commit(block *common.Block) { + panic(fmt.Errorf("Commit should not be invoked directly")) +} + +func (c *Chain) close() { + c.PeerLedger.Close() + c.m.wg.Done() +} + +func createLedgerByID(ledgerid string) (ledger.PeerLedger, error) { + gb, err := test.MakeGenesisBlock(ledgerid) + benchcommon.PanicOnError(err) + return ledgermgmt.CreateLedger(gb) +} diff --git a/test/tools/ledgerbenchmarks/chainmgmt/conf.go b/test/tools/ledgerbenchmarks/chainmgmt/conf.go new file mode 100644 index 00000000000..b24f020fad6 --- /dev/null +++ b/test/tools/ledgerbenchmarks/chainmgmt/conf.go @@ -0,0 +1,33 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chainmgmt + +// ChainMgrConf captures the configurations meant at the level of chainMgr +// DataDir field specifies the filesystem location where the chains data is maintained +// NumChains field specifies the number of chains to instantiate +type ChainMgrConf struct { + DataDir string + NumChains int +} + +// BatchConf captures the batch related configurations +// BatchSize specifies the number of transactions in one block +// SignBlock specifies whether the transactions should be signed +type BatchConf struct { + BatchSize int + SignBlock bool +} diff --git a/test/tools/ledgerbenchmarks/chainmgmt/testenv.go b/test/tools/ledgerbenchmarks/chainmgmt/testenv.go new file mode 100644 index 00000000000..f6ea89ad28f --- /dev/null +++ b/test/tools/ledgerbenchmarks/chainmgmt/testenv.go @@ -0,0 +1,62 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chainmgmt + +import "github.com/spf13/viper" + +// chainInitOp is a type that an experiment uses to specify how the chains +// should be initialized at the beginning of the experiment. See below the +// enum values for this type +type chainInitOp uint8 + +const ( + // ChainInitOpCreate indicates that the chains should be creates afresh + ChainInitOpCreate chainInitOp = iota + 1 + // ChainInitOpOpen indicates that the existing chains should be opened + ChainInitOpOpen +) + +// TestEnv is a high level struct that the experiments are expeted to use as a starting point. +// See one of the Benchmark tests for the intented usage +type TestEnv struct { + mgr *chainsMgr +} + +// InitTestEnv initialize TestEnv with given configurations. The initialization cuases +// creation (or openning of existing) chains and the block creation and commit go routines +// for each of the chains. For configurations options, see comments on specific configuration type +func InitTestEnv(mgrConf *ChainMgrConf, batchConf *BatchConf, initOperation chainInitOp) *TestEnv { + viper.Set("peer.fileSystemPath", mgrConf.DataDir) + mgr := newChainsMgr(mgrConf, batchConf, initOperation) + chains := mgr.createOrOpenChains() + for _, chain := range chains { + chain.startBlockPollingAndCommit() + } + return &TestEnv{mgr} +} + +// Chains returns handle to all the chains +func (env TestEnv) Chains() []*Chain { + return env.mgr.chains() +} + +// WaitForTestCompletion waits till all the transactions are committed +// An experiment after launching all the goroutine should call this +// so that the process is alive till all the goroutines complete +func (env TestEnv) WaitForTestCompletion() { + env.mgr.waitForChainsToExhaustAllBlocks() +} diff --git a/test/tools/ledgerbenchmarks/chainmgmt/tx_envelope_gen.go b/test/tools/ledgerbenchmarks/chainmgmt/tx_envelope_gen.go new file mode 100644 index 00000000000..6fab22405a9 --- /dev/null +++ b/test/tools/ledgerbenchmarks/chainmgmt/tx_envelope_gen.go @@ -0,0 +1,77 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chainmgmt + +import ( + mockmsp "github.com/hyperledger/fabric/common/mocks/msp" + "github.com/hyperledger/fabric/msp" + "github.com/hyperledger/fabric/protos/common" + pb "github.com/hyperledger/fabric/protos/peer" + putils "github.com/hyperledger/fabric/protos/utils" +) + +const ( + dummyChainID = "myChain" + dummyCCName = "myChaincode" + useDummyProposal = true +) + +var ( + dummyCCID = &pb.ChaincodeID{Name: dummyCCName, Version: "v1"} + dummyProposal *pb.Proposal + mspLcl msp.MSP + signer msp.SigningIdentity + serializedSigner []byte +) + +func init() { + mspLcl = mockmsp.NewNoopMsp() + signer, _ = mspLcl.GetDefaultSigningIdentity() + serializedSigner, _ = signer.Serialize() + + dummyProposal, _, _ = putils.CreateChaincodeProposal( + common.HeaderType_ENDORSER_TRANSACTION, dummyChainID, + &pb.ChaincodeInvocationSpec{ChaincodeSpec: &pb.ChaincodeSpec{ChaincodeId: dummyCCID}}, + serializedSigner) +} + +func createTxEnv(simulationResults []byte) (*common.Envelope, error) { + var prop *pb.Proposal + var err error + if useDummyProposal { + prop = dummyProposal + } else { + prop, _, err = putils.CreateChaincodeProposal( + common.HeaderType_ENDORSER_TRANSACTION, + dummyChainID, + &pb.ChaincodeInvocationSpec{ChaincodeSpec: &pb.ChaincodeSpec{ChaincodeId: dummyCCID}}, + serializedSigner) + if err != nil { + return nil, err + } + } + presp, err := putils.CreateProposalResponse(prop.Header, prop.Payload, nil, simulationResults, nil, dummyCCID, nil, signer) + if err != nil { + return nil, err + } + + env, err := putils.CreateSignedTx(prop, signer, presp) + if err != nil { + return nil, err + } + return env, nil +} diff --git a/test/tools/ledgerbenchmarks/common/util.go b/test/tools/ledgerbenchmarks/common/util.go new file mode 100644 index 00000000000..2175de4ef7d --- /dev/null +++ b/test/tools/ledgerbenchmarks/common/util.go @@ -0,0 +1,26 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import "fmt" + +// PanicOnError is an helper function to handle errors during experiment +func PanicOnError(err error) { + if err != nil { + panic(fmt.Errorf("Error:%s", err)) + } +} diff --git a/test/tools/ledgerbenchmarks/experiments/conf.go b/test/tools/ledgerbenchmarks/experiments/conf.go new file mode 100644 index 00000000000..ef2b274b491 --- /dev/null +++ b/test/tools/ledgerbenchmarks/experiments/conf.go @@ -0,0 +1,114 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiments + +import ( + "flag" + + "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/chainmgmt" +) + +// txConf captures the transaction related configurations +// numTotalTxs specifies the total transactions that should be executed and committed across chains +// numParallelTxsPerChain specifies the parallel transactions on each of the chains +// numKeysInEachTx specifies the number of keys that each of transactions should operate +type txConf struct { + numTotalTxs int + numParallelTxsPerChain int + numKeysInEachTx int +} + +// dataConf captures the data related configurations +// numKVs specifies number of total key-values across chains +// kvSize specifies the size of a key-value (in bytes) +type dataConf struct { + numKVs int + kvSize int +} + +// configuration captures all the configurations for an experiment +// For details of indivudual configuration, see comments on the specific type +type configuration struct { + chainMgrConf *chainmgmt.ChainMgrConf + batchConf *chainmgmt.BatchConf + dataConf *dataConf + txConf *txConf +} + +// defaultConf returns a configuration loaded with default values +func defaultConf() *configuration { + conf := &configuration{} + conf.chainMgrConf = &chainmgmt.ChainMgrConf{DataDir: "/tmp/fabric/ledgerPerfTests", NumChains: 1} + conf.batchConf = &chainmgmt.BatchConf{BatchSize: 10, SignBlock: false} + conf.txConf = &txConf{numTotalTxs: 100000, numParallelTxsPerChain: 100, numKeysInEachTx: 4} + conf.dataConf = &dataConf{numKVs: 100000, kvSize: 200} + return conf +} + +// emptyConf returns a an empty configuration (with nested structure only) +func emptyConf() *configuration { + conf := &configuration{} + conf.chainMgrConf = &chainmgmt.ChainMgrConf{} + conf.batchConf = &chainmgmt.BatchConf{} + conf.txConf = &txConf{} + conf.dataConf = &dataConf{} + return conf +} + +// confFromTestParams consumes the parameters passed by an experiment +// and returns the configuration loaded with the parsed param values +func confFromTestParams(testParams []string) *configuration { + conf := emptyConf() + flags := flag.NewFlagSet("testParams", flag.ExitOnError) + + // chainMgrConf + dataDir := flags.String("DataDir", conf.chainMgrConf.DataDir, "Dir for ledger data") + numChains := flags.Int("NumChains", conf.chainMgrConf.NumChains, "Number of chains") + + // txConf + numParallelTxsPerChain := flags.Int("NumParallelTxPerChain", + conf.txConf.numParallelTxsPerChain, "Number of TxSimulators concurrently on each chain") + + numTotalTxs := flags.Int("NumTotalTx", + conf.txConf.numTotalTxs, "Number of total transactions") + + numKeysInEachTx := flags.Int("NumKeysInEachTx", + conf.txConf.numKeysInEachTx, "number of keys operated upon in each Tx") + + // batchConf + batchSize := flags.Int("BatchSize", + conf.batchConf.BatchSize, "number of Txs in each batch") + + // dataConf + numKVs := flags.Int("NumKVs", + conf.dataConf.numKVs, "the keys are named as key_0, key_1,... upto key_(NumKVs-1)") + + kvSize := flags.Int("KVSize", + conf.dataConf.kvSize, "size of the key-value in bytes") + + flags.Parse(testParams) + + conf.chainMgrConf.DataDir = *dataDir + conf.chainMgrConf.NumChains = *numChains + conf.txConf.numParallelTxsPerChain = *numParallelTxsPerChain + conf.txConf.numTotalTxs = *numTotalTxs + conf.txConf.numKeysInEachTx = *numKeysInEachTx + conf.batchConf.BatchSize = *batchSize + conf.dataConf.numKVs = *numKVs + conf.dataConf.kvSize = *kvSize + return conf +} diff --git a/test/tools/ledgerbenchmarks/experiments/init_test.go b/test/tools/ledgerbenchmarks/experiments/init_test.go new file mode 100644 index 00000000000..e5dbfca750c --- /dev/null +++ b/test/tools/ledgerbenchmarks/experiments/init_test.go @@ -0,0 +1,60 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiments + +import ( + "flag" + "os" + "regexp" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/hyperledger/fabric/core/ledger/testutil" + + "fmt" +) + +const chaincodeName = "testChaincode" + +var conf *configuration + +// TestMain loads the yaml file and parses the test parameters +// the configration constructed from test parameters is stored in +// package level variable and is accessible to an expriment +// The test params should be passed in the following format in the golang benchmark test command +// "-testParams=-DataDir="myDir", -NumChains=10, ..." +// This is necessary to parse in the TestMain function because otherwise, the golang test framework +// parses this and does not recognized this flag (-testParams) +func TestMain(m *testing.M) { + testutil.SetupCoreYAMLConfig() + testParams := parseTestParams() + conf = confFromTestParams(testParams) + logger.Infof("Running experiment with configuration: %s\n", spew.Sdump(conf)) + disableLogging() + os.Exit(m.Run()) +} + +func parseTestParams() []string { + testParams := flag.String("testParams", "", "Test specific parameters") + flag.Parse() + regex, err := regexp.Compile(",(\\s+)?") + if err != nil { + panic(fmt.Errorf("Error: %s", err)) + } + paramsArray := regex.Split(*testParams, -1) + return paramsArray +} diff --git a/test/tools/ledgerbenchmarks/experiments/insert_txs_test.go b/test/tools/ledgerbenchmarks/experiments/insert_txs_test.go new file mode 100644 index 00000000000..905f638a24d --- /dev/null +++ b/test/tools/ledgerbenchmarks/experiments/insert_txs_test.go @@ -0,0 +1,88 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiments + +import ( + "sync" + "testing" + + "fmt" + + "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/chainmgmt" + "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common" +) + +// BenchmarkInsertTxs starts fresh chains and inserts the Key-values by simulating writes-only transactions +// For each of the chains, this test launches the parallel clients (based on the configuration) and the clients +// simulate and commit write-only transactions. The keys are divided among clients such that one key is written +// only once and all the keys are inserted. +// +// For instance, if this benchmark is invoked with the following test parameters +// -testParams=-NumChains=2, -NumParallelTxPerChain=2, -NumKVs=100 +// then client_1 on chain_1 will insert Key_1 to key_25 and client_2 on chain_1 will insert Key_26 to key_50 +// similarly client_3 on chain_2 will insert Key_1 to key_25 and client_4 on chain_2 will insert Key_26 to key_50 +// where, client_1 and client_2 run in parallel on chain_1 and client_3 and client_4 run in parallel on chain_2 +func BenchmarkInsertTxs(b *testing.B) { + if b.N != 1 { + panic(fmt.Errorf(`This benchmark should be called with N=1 only. Run this with more volume of data`)) + } + testEnv := chainmgmt.InitTestEnv(conf.chainMgrConf, conf.batchConf, chainmgmt.ChainInitOpCreate) + for _, chain := range testEnv.Chains() { + go runInsertClientsForChain(chain) + } + testEnv.WaitForTestCompletion() +} + +func runInsertClientsForChain(chain *chainmgmt.Chain) { + numKVsForChain := calculateShare(conf.dataConf.numKVs, conf.chainMgrConf.NumChains, int(chain.ID)) + numClients := conf.txConf.numParallelTxsPerChain + wg := &sync.WaitGroup{} + wg.Add(numClients) + startKey := 0 + for i := 0; i < numClients; i++ { + numKVsForClient := calculateShare(numKVsForChain, numClients, i) + endKey := startKey + numKVsForClient - 1 + go runInsertClient(chain, startKey, endKey, wg) + startKey = endKey + 1 + } + wg.Wait() + chain.Done() +} + +func runInsertClient(chain *chainmgmt.Chain, startKey, endKey int, wg *sync.WaitGroup) { + numKeysPerTx := conf.txConf.numKeysInEachTx + kvSize := conf.dataConf.kvSize + + currentKey := startKey + for currentKey <= endKey { + simulator, err := chain.NewTxSimulator() + common.PanicOnError(err) + for i := 0; i < numKeysPerTx; i++ { + common.PanicOnError(simulator.SetState( + chaincodeName, constructKey(currentKey), constructValue(currentKey, kvSize))) + currentKey++ + if currentKey > endKey { + break + } + } + simulator.Done() + sr, err := simulator.GetTxSimulationResults() + common.PanicOnError(err) + chain.SubmitTx(sr) + } + wg.Done() +} diff --git a/test/tools/ledgerbenchmarks/experiments/readwrite_txs_test.go b/test/tools/ledgerbenchmarks/experiments/readwrite_txs_test.go new file mode 100644 index 00000000000..ae847ed8c40 --- /dev/null +++ b/test/tools/ledgerbenchmarks/experiments/readwrite_txs_test.go @@ -0,0 +1,90 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiments + +import ( + "fmt" + "math/rand" + "sync" + "testing" + "time" + + "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/chainmgmt" + "github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/common" +) + +// BenchmarkReadWriteTxs opens the existing chains and modifies the Key-values by simulating read-write transactions +// For each of the chains, this test launches the parallel clients (based on the configuration) and the clients +// simulate and commit read-write transactions. This test assumes the pre-populated chain by previously running +// BenchmarkInsertTxs. Each transaction simultated by this benchmark randomly selects a configurable number of keys +// and modifies their values +// +// For instance, if this benchmark is invoked with the following test parameters +// -testParams=-NumChains=2, -NumParallelTxPerChain=2, -NumKVs=100, -NumTotalTx=200 +// then client_1, and client_2 both execute 50 transactions on chain_1 in parallel. Similarly, +// client_3, and client_4 both execute 50 transactions on chain_2 in parallel +// In each of the transactions executed by any client, the transaction expects +// and modifies any key(s) between Key_1 to key_50 (because, total keys are to be 100 across two chains) +func BenchmarkReadWriteTxs(b *testing.B) { + if b.N != 1 { + panic(fmt.Errorf(`This benchmark should be called with N=1 only. Run this with more volume of data`)) + } + testEnv := chainmgmt.InitTestEnv(conf.chainMgrConf, conf.batchConf, chainmgmt.ChainInitOpOpen) + for _, chain := range testEnv.Chains() { + go runReadWriteClientsForChain(chain) + } + testEnv.WaitForTestCompletion() +} + +func runReadWriteClientsForChain(chain *chainmgmt.Chain) { + numClients := conf.txConf.numParallelTxsPerChain + numTxForChain := calculateShare(conf.txConf.numTotalTxs, conf.chainMgrConf.NumChains, int(chain.ID)) + wg := &sync.WaitGroup{} + wg.Add(numClients) + for i := 0; i < numClients; i++ { + numTxForClient := calculateShare(numTxForChain, numClients, i) + randomNumGen := rand.New(rand.NewSource(int64(time.Now().Nanosecond()) + int64(chain.ID))) + go runReadWriteClient(chain, randomNumGen, numTxForClient, wg) + } + wg.Wait() + chain.Done() +} + +func runReadWriteClient(chain *chainmgmt.Chain, rand *rand.Rand, numTx int, wg *sync.WaitGroup) { + numKeysPerTx := conf.txConf.numKeysInEachTx + maxKeyNumber := calculateShare(conf.dataConf.numKVs, conf.chainMgrConf.NumChains, int(chain.ID)) + + for i := 0; i < numTx; i++ { + simulator, err := chain.NewTxSimulator() + common.PanicOnError(err) + for i := 0; i < numKeysPerTx; i++ { + keyNumber := rand.Intn(maxKeyNumber) + key := constructKey(keyNumber) + value, err := simulator.GetState(chaincodeName, key) + common.PanicOnError(err) + if !verifyValue(keyNumber, value) { + panic(fmt.Errorf("Value %s is not expected for key number %d", value, keyNumber)) + } + common.PanicOnError(simulator.SetState(chaincodeName, key, value)) + } + simulator.Done() + sr, err := simulator.GetTxSimulationResults() + common.PanicOnError(err) + chain.SubmitTx(sr) + } + wg.Done() +} diff --git a/test/tools/ledgerbenchmarks/experiments/util.go b/test/tools/ledgerbenchmarks/experiments/util.go new file mode 100644 index 00000000000..f85ca32e9d5 --- /dev/null +++ b/test/tools/ledgerbenchmarks/experiments/util.go @@ -0,0 +1,68 @@ +/* +Copyright IBM Corp. 2017 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiments + +import ( + "bytes" + "fmt" + "math/rand" + + logging "github.com/op/go-logging" +) + +var logger = logging.MustGetLogger("experiments") + +func constructKey(keyNumber int) string { + return fmt.Sprintf("%s%09d", "key_", keyNumber) +} + +func constructValue(keyNumber int, kvSize int) []byte { + prefix := constructValuePrefix(keyNumber) + randomBytes := constructRandomBytes(kvSize - len(prefix)) + return append(prefix, randomBytes...) +} + +func constructValuePrefix(keyNumber int) []byte { + return []byte(fmt.Sprintf("%s%09d", "value_", keyNumber)) +} + +func verifyValue(keyNumber int, value []byte) bool { + prefix := constructValuePrefix(keyNumber) + if len(value) < len(prefix) { + return false + } + return bytes.Equal(value[:len(prefix)], prefix) +} + +func disableLogging() { + logging.SetLevel(logging.ERROR, "") +} + +func calculateShare(total int, numParts int, partNum int) int { + share := total / numParts + remainder := total % numParts + if partNum < remainder { + share++ + } + return share +} + +func constructRandomBytes(length int) []byte { + b := make([]byte, length) + rand.Read(b) + return b +} diff --git a/test/tools/ledgerbenchmarks/scripts/benchmarks.sh b/test/tools/ledgerbenchmarks/scripts/benchmarks.sh new file mode 100755 index 00000000000..dfaf2bec0e2 --- /dev/null +++ b/test/tools/ledgerbenchmarks/scripts/benchmarks.sh @@ -0,0 +1,37 @@ +#!/bin/bash +source ./common.sh + +####################################################################################################### +# This shell script contains two functions that can be invoked to execute specific tests +# +# runInsertTxs - This function sets the environment variables and runs the benchmark function +# 'BenchmarkInsertTxs' in package 'github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/experiments' +# +# runReadWriteTxs - This function sets the environment variables and runs the benchmark function +# 'BenchmarkReadWriteTxs' in package 'github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/experiments' +# +# For the details of test specific parameters, refer to the documentation in 'go' files for the tests +####################################################################################################### + +PKG_NAME="github.com/hyperledger/fabric/test/tools/ledgerbenchmarks/experiments" + +function setCommonTestParams { + TEST_PARAMS="-DataDir=$DataDir, -NumChains=$NumChains, -NumParallelTxPerChain=$NumParallelTxPerChain, -NumKeysInEachTx=$NumKeysInEachTx, -BatchSize=$BatchSize, -NumKVs=$NumKVs, -KVSize=$KVSize" + RESULTANT_DIRS="$DataDir/ledgersData/chains/chains $DataDir/ledgersData/chains/index $DataDir/ledgersData/stateLeveldb $DataDir/ledgersData/historyLeveldb" +} + +function runInsertTxs { + FUNCTION_NAME="BenchmarkInsertTxs" + setCommonTestParams + executeTest +} + +function runReadWriteTxs { + FUNCTION_NAME="BenchmarkReadWriteTxs" + if [ "$CLEAR_OS_CACHE" == "true" ]; then + clearOSCache + fi + setCommonTestParams + TEST_PARAMS="$TEST_PARAMS, -NumTotalTx=$NumTotalTx" + executeTest +} \ No newline at end of file diff --git a/test/tools/ledgerbenchmarks/scripts/common.sh b/test/tools/ledgerbenchmarks/scripts/common.sh new file mode 100755 index 00000000000..3a9da4650e8 --- /dev/null +++ b/test/tools/ledgerbenchmarks/scripts/common.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +set -e + +############################################################################################################################# +# This shell script contains common functions that can be used across benchmark tests +# Following is the description of the list of variables that this script uses +# +# OUTPUT_DIR_ROOT - Root dir for tests results +# RAW_OUTPUT_FILE - File name that contains the raw output produced by a test that otherwiese is printed on screen +# RESULTS_FILE - File name that contains the test parameters and the time taken by the test (in csv format) +# PKG_NAME - Name of the golang package for the test +# FUNCTION_NAME - Name of the Benchmark function +# TEST_PARAMS - Parameters for the test +# RESULTANT_DIRS - An optional list of dirs whose size needs to be captured in the results in the RESULTS_FILE +# +# The default values for some of the above variables are set in this script and can be overrideen by a test specific +# script. For remaining variables, a test specific script needs to set the appropriate values before calling the +# function 'executeTest' of this script +# +# The result file for a test gets created in a csv format in the folder +# $OUTPUT_DIR_ROOT/$PKG_NAME/$FUNCTION_NAME +############################################################################################################################# + + +OUTPUT_DIR_ROOT=`echo /tmp/fabric/test/tools/ledgerbenchmarks/results` +RAW_OUTPUT_FILE="output.txt" +RESULTS_FILE="results.csv" + +benchmarkLineRegex="^Benchmark.*[[:blank:]]+[[:digit:]]+[[:blank:]]+([[:digit:]]+).*$" +testParamRegex="-\([^=]*\)=\([^,]*\)" + +ulimit -n 10000 +echo "ulimit=`ulimit -n`" +TESTS_SETUP_DONE=() + +## Execute test and generate data file +function executeTest { + runTestSetup + cmd="go test -v -timeout 1000m $PKG_NAME -testParams=\"$TEST_PARAMS\" -bench=$FUNCTION_NAME" + echo $cmd + RAW_OUTPUT=`eval $cmd` + writeResults +} + +function writeResults { + outputDir=`getOuputDir` + echo "Test Output Start:" + echo "$RAW_OUTPUT" + echo "Test Output Finish" + while read -r line; do + echo $line >> $outputDir/$RAW_OUTPUT_FILE + if [[ $line =~ $benchmarkLineRegex ]]; then + resultsDataLine="`extractParamValues`, `nanosToSec ${BASH_REMATCH[1]}`" + for d in $RESULTANT_DIRS; do + dirSizeKBs=`du -sk $d | cut -f1` + resultsDataLine="$resultsDataLine, `kbsToMbs $dirSizeKBs`" + done + echo $resultsDataLine >> $outputDir/$RESULTS_FILE + fi + done <<< "$RAW_OUTPUT" +} + +function runTestSetup { + outputDir=`getOuputDir` + for d in ${TESTS_SETUP_DONE[@]} + do + if [ $d == $outputDir ] + then + return + fi + done + createOutputDir + writeResultsFileHeader + TESTS_SETUP_DONE+=($outputDir) +} + +function writeResultsFileHeader { + outputDir=`getOuputDir` + echo "" >> $outputDir/$RESULTS_FILE + echo "# `date`" >> $outputDir/$RESULTS_FILE + headerLine="# `extractParamNames`, Time_Spent(s)" + for d in $RESULTANT_DIRS; do + headerLine="$headerLine, Size_$(basename $d)(mb)" + done + echo "$headerLine" >> $outputDir/$RESULTS_FILE +} + +function extractParamNames { + echo $TEST_PARAMS | sed "s/$testParamRegex/\1/g" +} + +function extractParamValues { + echo $TEST_PARAMS | sed "s/$testParamRegex/\2/g" +} + +function getOuputDir { + pkgName=$(basename $PKG_NAME) + outputDir="$OUTPUT_DIR_ROOT/$pkgName/$FUNCTION_NAME" + if [ ! -z "$OUTPUT_DIR" ]; then + outputDir="$OUTPUT_DIR_ROOT/$pkgName/$OUTPUT_DIR" + fi + echo $outputDir +} + +function createOutputDir { + outputDir=`getOuputDir` + if [ ! -d "$outputDir" ]; then + mkdir -p $outputDir + else + echo "INFO: outputDIR [$outputDir] already exists. Output will be appended to existing file" + fi +} + +function clearOSCache { + platform=`uname` + if [[ $platform == 'Darwin' ]]; then + echo "Clearing os cache" + sudo purge + else + echo "WARNING: Platform [$platform] is not supported for clearing os cache." + fi +} + +function nanosToSec { + nanos=$1 + echo $(awk "BEGIN {printf \"%.2f\", ${nanos}/1000000000}") +} + +function kbsToMbs { + kbs=$1 + echo $(awk "BEGIN {printf \"%.2f\", ${kbs}/1024}") +} \ No newline at end of file diff --git a/test/tools/ledgerbenchmarks/scripts/runbenchmarks.sh b/test/tools/ledgerbenchmarks/scripts/runbenchmarks.sh new file mode 100755 index 00000000000..9d690057410 --- /dev/null +++ b/test/tools/ledgerbenchmarks/scripts/runbenchmarks.sh @@ -0,0 +1,101 @@ +#!/bin/bash +source ./benchmarks.sh + +######################################################################################################## +# This shell script invokes a series of benchmark tests with desired values of test specific parameters +######################################################################################################## + +function setDefaultTestParams { + DataDir="/tmp/fabric/test/tools/ledgerbenchmarks/data" + NumChains=10 + NumParallelTxPerChain=10 + NumKVs=1000000 + NumTotalTx=1000000 + NumKeysInEachTx=4 + BatchSize=50 + KVSize=200 +} + +function varyNumParallelTxPerChain { + setDefaultTestParams + for v in 1 5 10 20 50 100 500 2000; do + NumParallelTxPerChain=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function varyNumChains { + setDefaultTestParams + for v in 1 5 10 20 50 100 500 2000; do + NumChains=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function varyNumKeysInEachTx { + setDefaultTestParams + for v in 1 2 5 10 20; do + NumKeysInEachTx=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function varyKVSize { + setDefaultTestParams + for v in 100 200 500 1000 2000; do + KVSize=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function varyBatchSize { + setDefaultTestParams + for v in 10 20 100 500; do + BatchSize=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function varyNumParallelTxWithSingleChain { + setDefaultTestParams + NumChains=1 + for v in 1 5 10 20 50 100 500 2000; do + NumParallelTxPerChain=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function varyNumChainsWithNoParallelism { + setDefaultTestParams + NumParallelTxPerChain=1 + for v in 1 5 10 20 50 100 500 2000; do + NumChains=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function varyNumTxs { + setDefaultTestParams + for v in 1000000 2000000 5000000 10000000; do + NumTotalTx=$v + rm -rf $DataDir;runInsertTxs;runReadWriteTxs + done +} + +function runLargeDataExperiment { + setDefaultTestParams + NumKVs=10000000 + NumTotalTx=10000000 + rm -rf $DataDir;runInsertTxs;runReadWriteTxs +} + +### Run tests +varyNumParallelTxPerChain +varyNumChains +varyNumParallelTxWithSingleChain +varyNumChainsWithNoParallelism +varyNumKeysInEachTx +varyKVSize +varyBatchSize +varyNumTxs +runLargeDataExperiment \ No newline at end of file