From a5ad73460880b6cd81495c65e3786f0c098b3850 Mon Sep 17 00:00:00 2001 From: Adnan Choudhury Date: Wed, 24 May 2017 12:06:47 -0400 Subject: [PATCH] [FAB-3749] Add Ledger Component Perf. Tests Adding Ledger component performance tests to the test framework. The tests use the LTE tool. The attached Docstring to each test explains what the test would perform. This includes the following tests: [FAB-3790] [FAB-3795] [FAB-3798] [FAB-3799] [FAB-3801] [FAB-3802] [FAB-3800] [FAB-3803] These tests can be run by typing: $ py.test -v --junitxml results_ledger_lte.xml ledger_lte.py Change-Id: I92fab479518131305be66932ef2d60760e1ab9e7 Signed-off-by: Adnan Choudhury --- test/regression/daily/ledger_lte.py | 206 +++++++++++++++++++++ test/regression/daily/runDailyTestSuite.sh | 16 +- 2 files changed, 208 insertions(+), 14 deletions(-) create mode 100644 test/regression/daily/ledger_lte.py diff --git a/test/regression/daily/ledger_lte.py b/test/regression/daily/ledger_lte.py new file mode 100644 index 00000000000..2601e45e2b2 --- /dev/null +++ b/test/regression/daily/ledger_lte.py @@ -0,0 +1,206 @@ +import unittest +import subprocess + +class perf_goleveldb(unittest.TestCase): + + def test_FAB_3790_VaryNumParallelTxPerChain(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of parallel + transactions per chain and observe the performance. + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumParallelTxPerChain", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3795_VaryNumChain(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of chains + (ledgers). + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumChain", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3798_VaryNumParallelTxWithSingleChain(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of parallel + transactions on a single chain. + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumParallelTxWithSingleChain", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3799_VaryNumChainWithNoParallelism(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of chains + without any parallelism within a single chain. + + Passing criteria: all subtests (8) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumChainWithNoParallelism", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 8) + + def test_FAB_3801_VaryKVSize(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the size of key-value. + + Passing criteria: all subtests (5) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyKVSize", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 5) + + def test_FAB_3802_VaryBatchSize(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the value of the batch + size + + Passing criteria: all subtests (4) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyBatchSize", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 4) + + def test_FAB_3800_VaryNumKeysInEachTX(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of keys in + each transaction. + + Passing criteria: all subtests (5) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumKeysInEachTX", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 5) + + def test_FAB_3803_VaryNumTxs(self): + ''' + In this Performance test, we observe the the performance (time to + complete a set number of Ledger operations) of the Ledger component, + with goleveldb as the state database. We vary the number of + transactions carried out. + + Passing criteria: all subtests (4) completed successfully + ''' + result = subprocess.check_output( + "./runbenchmarks.sh varyNumTxs", + shell=True, stderr=subprocess.STDOUT, + cwd='../../tools/LTE/scripts') + completion_count = result.count("PASS") + self.assertEqual(completion_count, 4) + + +class perf_couchdb(unittest.TestCase): + @unittest.skip("WIP, skipping") + def test_FAB_3870_VaryNumParallelTxPerChain(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of parallel transactions per chain. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3871_VaryNumChain(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of chains (ledgers). + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3872_VaryNumParallelTxWithSingleChain(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, vary the number of parallel transactions on a single chain. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3873_VaryNumChainWithNoParallelism(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of chains without any parallelism. + within a single chain. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3874_VaryKVSize(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, varying the size of key-value. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3875_VaryBatchSize(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the value of the batch size. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3876_VaryNumKeysInEachTX(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of keys in each transaction. + ''' + self.assertTrue(True) + + @unittest.skip("WIP, skipping") + def test_FAB_3877_VaryNumTxs(self): + ''' + In this Performance test, we observe the the performance (operations + per second) of the Ledger component, with CouchDB as the state + database, as we vary the number of transactions carried out. + ''' + self.assertTrue(True) diff --git a/test/regression/daily/runDailyTestSuite.sh b/test/regression/daily/runDailyTestSuite.sh index c0a4bd217d9..a0beb217634 100755 --- a/test/regression/daily/runDailyTestSuite.sh +++ b/test/regression/daily/runDailyTestSuite.sh @@ -19,17 +19,5 @@ cd ../../feature behave --junit --junit-directory ../regression/daily/. --tags=-skip --tags=daily cd - -echo "========== Test Your Chaincode ..." -# TBD - after changeset https://gerrit.hyperledger.org/r/#/c/9163/ is merged, -# replace the previous 2 lines with this new syntax to run all the chaincode tests; -# and when making this change we should also remove file chaincodeTests/runChaincodes.sh) -# -#cd $DAILYDIR/chaincodeTests/envsetup -#py.test -v --junitxml ../../results_testYourChaincode.xml testYourChaincode.py - -# TBD - after changeset https://gerrit.hyperledger.org/r/#/c/9251/ is merged, -# and integrated with this, lines like these should be executed too: -#echo "========== Ledger component performance tests..." -#cd $DAILYDIR/ledgerperftests -#py.test -v --junitxml results_perf_goleveldb.xml test_perf_goleveldb.py -#py.test -v --junitxml results_perf_couchdb.xml test_perf_couchdb.py +echo "========== Ledger component performance tests..." +py.test -v --junitxml results_ledger_lte.xml ledger_lte.py