diff --git a/test/regression/daily/ledger_lte.py b/test/regression/daily/ledger_lte.py new file mode 100644 index 00000000000..2601e45e2b2 --- /dev/null +++ b/test/regression/daily/ledger_lte.py @@ -0,0 +1,206 @@ +import unittest +import subprocess + +class perf_goleveldb(unittest.TestCase): + + def test_FAB_3790_VaryNumParallelTxPerChain(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of parallel + transactions per chain and observe the performance. + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumParallelTxPerChain", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3795_VaryNumChain(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of chains + (ledgers). + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumChain", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3798_VaryNumParallelTxWithSingleChain(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of parallel + transactions on a single chain. + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumParallelTxWithSingleChain", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3799_VaryNumChainWithNoParallelism(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of chains + without any parallelism within a single chain. + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumChainWithNoParallelism", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3801_VaryKVSize(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the size of key-value. + + Passing criteria: all subtests (5) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyKVSize", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 5) + + def test_FAB_3802_VaryBatchSize(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the value of the batch + size + + Passing criteria: all subtests (4) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyBatchSize", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 4) + + def test_FAB_3800_VaryNumKeysInEachTX(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of keys in + each transaction. + + Passing criteria: all subtests (5) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumKeysInEachTX", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 5) + + def test_FAB_3803_VaryNumTxs(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of + transactions carried out. + + Passing criteria: all subtests (4) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumTxs", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 4) + + +class perf_couchdb(unittest.TestCase): + @unittest.skip("WIP, skipping") + def test_FAB_3870_VaryNumParallelTxPerChain(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of parallel transactions per chain. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3871_VaryNumChain(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of chains (ledgers). + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3872_VaryNumParallelTxWithSingleChain(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, vary the number of parallel transactions on a single chain. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3873_VaryNumChainWithNoParallelism(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of chains without any parallelism. + within a single chain. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3874_VaryKVSize(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, varying the size of key-value. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3875_VaryBatchSize(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the value of the batch size. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3876_VaryNumKeysInEachTX(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of keys in each transaction. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3877_VaryNumTxs(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of transactions carried out. + ''' + self.assertTrue(True) diff --git a/test/regression/daily/runDailyTestSuite.sh b/test/regression/daily/runDailyTestSuite.sh index c0a4bd217d9..a0beb217634 100755 --- a/test/regression/daily/runDailyTestSuite.sh +++ b/test/regression/daily/runDailyTestSuite.sh @@ -19,17 +19,5 @@ cd ../../feature behave --junit --junit-directory ../regression/daily/. --tags=-skip --tags=daily cd - -echo "========== Test Your Chaincode ..." -# TBD - after changeset https://gerrit.hyperledger.org/r/#/c/9163/ is merged, -# replace the previous 2 lines with this new syntax to run all the chaincode tests; -# and when making this change we should also remove file chaincodeTests/runChaincodes.sh) -# -#cd $DAILYDIR/chaincodeTests/envsetup -#py.test -v --junitxml ../../results_testYourChaincode.xml testYourChaincode.py - -# TBD - after changeset https://gerrit.hyperledger.org/r/#/c/9251/ is merged, -# and integrated with this, lines like these should be executed too: -#echo "========== Ledger component performance tests..." -#cd $DAILYDIR/ledgerperftests -#py.test -v --junitxml results_perf_goleveldb.xml test_perf_goleveldb.py -#py.test -v --junitxml results_perf_couchdb.xml test_perf_couchdb.py +echo "========== Ledger component performance tests..." +py.test -v --junitxml results_ledger_lte.xml ledger_lte.py