Merge "[FAB-3749] Add Ledger Component Perf. Tests"

test/regression/daily/ledger_lte.py

@@ -0,0 +1,206 @@
+import unittest
+import subprocess
+
+class perf_goleveldb(unittest.TestCase):
+
+    def test_FAB_3790_VaryNumParallelTxPerChain(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the number of parallel
+         transactions per chain and observe the performance.
+
+         Passing criteria: all subtests (8) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyNumParallelTxPerChain",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 8)
+
+    def test_FAB_3795_VaryNumChain(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the number of chains
+         (ledgers).
+
+         Passing criteria: all subtests (8) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyNumChain",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 8)
+
+    def test_FAB_3798_VaryNumParallelTxWithSingleChain(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the number of parallel
+         transactions on a single chain.
+
+         Passing criteria: all subtests (8) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyNumParallelTxWithSingleChain",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 8)
+
+    def test_FAB_3799_VaryNumChainWithNoParallelism(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the number of chains
+         without any parallelism within a single chain.
+
+         Passing criteria: all subtests (8) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyNumChainWithNoParallelism",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 8)
+
+    def test_FAB_3801_VaryKVSize(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the size of key-value.
+
+         Passing criteria: all subtests (5) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyKVSize",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 5)
+
+    def test_FAB_3802_VaryBatchSize(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the value of the batch
+         size
+
+         Passing criteria: all subtests (4) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyBatchSize",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 4)
+
+    def test_FAB_3800_VaryNumKeysInEachTX(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the number of keys in
+         each transaction.
+
+         Passing criteria: all subtests (5) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyNumKeysInEachTX",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 5)
+
+    def test_FAB_3803_VaryNumTxs(self):
+        '''
+         In this Performance test, we observe the the performance (time to
+         complete a set number of Ledger operations) of the Ledger component,
+         with goleveldb as the state database. We vary the number of
+         transactions carried out.
+
+         Passing criteria: all subtests (4) completed successfully
+        '''
+        result = subprocess.check_output(
+                "./runbenchmarks.sh varyNumTxs",
+                shell=True, stderr=subprocess.STDOUT,
+                cwd='../../tools/LTE/scripts')
+        completion_count = result.count("PASS")
+        self.assertEqual(completion_count, 4)
+
+
+class perf_couchdb(unittest.TestCase):
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3870_VaryNumParallelTxPerChain(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, as we vary the number of parallel transactions per chain.
+        '''
+        self.assertTrue(True)
+
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3871_VaryNumChain(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, as we vary the number of chains (ledgers).
+        '''
+        self.assertTrue(True)
+
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3872_VaryNumParallelTxWithSingleChain(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, vary the number of parallel transactions on a single chain.
+        '''
+        self.assertTrue(True)
+
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3873_VaryNumChainWithNoParallelism(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, as we vary the number of chains without any parallelism.
+         within a single chain.
+        '''
+        self.assertTrue(True)
+
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3874_VaryKVSize(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, varying the size of key-value.
+        '''
+        self.assertTrue(True)
+
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3875_VaryBatchSize(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, as we vary the value of the batch size.
+        '''
+        self.assertTrue(True)
+
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3876_VaryNumKeysInEachTX(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, as we vary the number of keys in each transaction.
+        '''
+        self.assertTrue(True)
+
+    @unittest.skip("WIP, skipping")
+    def test_FAB_3877_VaryNumTxs(self):
+        '''
+         In this Performance test, we observe the the performance (operations
+         per second) of the Ledger component, with CouchDB as the state
+         database, as we vary the number of transactions carried out.
+        '''
+        self.assertTrue(True)

test/regression/daily/runDailyTestSuite.sh

@@ -19,17 +19,5 @@ cd ../../feature
 behave --junit --junit-directory ../regression/daily/. --tags=-skip --tags=daily
 cd -
 
-echo "========== Test Your Chaincode ..."
-# TBD - after changeset https://gerrit.hyperledger.org/r/#/c/9163/ is merged,
-# replace the previous 2 lines with this new syntax to run all the chaincode tests;
-# and when making this change we should also remove file chaincodeTests/runChaincodes.sh)
-#
-#cd $DAILYDIR/chaincodeTests/envsetup
-#py.test -v --junitxml ../../results_testYourChaincode.xml testYourChaincode.py
-
-# TBD - after changeset https://gerrit.hyperledger.org/r/#/c/9251/ is merged,
-# and integrated with this, lines like these should be executed too:
-#echo "========== Ledger component performance tests..."
-#cd $DAILYDIR/ledgerperftests
-#py.test -v --junitxml results_perf_goleveldb.xml test_perf_goleveldb.py
-#py.test -v --junitxml results_perf_couchdb.xml test_perf_couchdb.py
+echo "========== Ledger component performance tests..."
+py.test -v --junitxml results_ledger_lte.xml ledger_lte.py